diff --git a/feeds/ipq807x/ipq807x/Makefile b/feeds/ipq807x/ipq807x/Makefile new file mode 100644 index 000000000..abdb82ec5 --- /dev/null +++ b/feeds/ipq807x/ipq807x/Makefile @@ -0,0 +1,22 @@ +include $(TOPDIR)/rules.mk + +ARCH:=arm +BOARD:=ipq807x +BOARDNAME:=Qualcomm Atheros AX +SUBTARGETS:=ipq807x ipq60xx ipq50xx +FEATURES:=squashfs ramdisk nand pcie usb +KERNELNAME:=Image dtbs +CPU_TYPE:=cortex-a7 + +KERNEL_PATCHVER:=4.4 +KERNEL_NAME_SUFFIX=-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016 + +include $(INCLUDE_DIR)/target.mk +DEFAULT_PACKAGES += kmod-qca-nss-dp kmod-qca-ssdk swconfig \ + kmod-qca-nss-drv \ + kmod-usb-phy-ipq807x kmod-usb-dwc3-of-simple \ + kmod-ath11k-ahb kmod-qrtr_mproc wpad \ + kmod-gpio-button-hotplug \ + qca-thermald-10.4 qca-ssdk-shell kmod-qca-nss-drv-bridge-mgr + +$(eval $(call BuildTarget)) diff --git a/feeds/ipq807x/ipq807x/base-files/etc/board.d/01_leds b/feeds/ipq807x/ipq807x/base-files/etc/board.d/01_leds new file mode 100755 index 000000000..d47a7e55e --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/board.d/01_leds @@ -0,0 +1,44 @@ +#!/bin/sh + +. /lib/functions/uci-defaults.sh + +board_config_update + +board=$(board_name) + +case "$board" in +sercomm,wallaby|\ +wallys,dr6018|\ +wallys,dr6018-v4|\ +cig,wf188n|\ +cig,wf194c|\ +cig,wf194c4) + ucidef_set_led_netdev "wan" "WAN" "green:wan" "eth0" "tx rx link" + ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" + ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" + ;; +esac + +case "$board" in +edgecore,eap101) + ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" + ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" + ucidef_set_led_netdev "lan1" "lan1" "green:lan1" "eth1" + ucidef_set_led_netdev "lan2" "lan2" "green:lan2" "eth2" + ucidef_set_led_netdev "poe" "poe" "green:wan" "eth0" + ;; +edgecore,eap102) + ucidef_set_led_netdev "poe" "poe" "green:wan" "eth0" + ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" + ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" + ;; +hfcl,ion4xi|\ +hfcl,ion4xe) + ucidef_set_led_wlan "wlan5g" "WLAN5G" "blue:wifi5" "phy0tpt" + ucidef_set_led_wlan "wlan2g" "WLAN2G" "blue:wifi2" "phy1tpt" + ;; +esac + +board_config_flush + +exit 0 diff --git a/feeds/ipq807x/ipq807x/base-files/etc/board.d/02_network b/feeds/ipq807x/ipq807x/base-files/etc/board.d/02_network new file mode 100755 index 000000000..6e8dc9b01 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/board.d/02_network @@ -0,0 +1,91 @@ +#!/bin/sh + +. /lib/functions.sh +. /lib/functions/uci-defaults.sh +. /lib/functions/system.sh + +qcom_setup_interfaces() +{ + local board="$1" + ucidef_add_switch "switch0" + ucidef_add_switch_attr "switch0" "enable" "false" + ucidef_add_switch_attr "switch0" "reset" "false" + + case $board in + hfcl,ion4xi|\ + hfcl,ion4xe) + ucidef_set_interface_wan "eth0 eth1" + ucidef_set_interface_lan "" + ;; + cig,wf188|\ + tplink,ex227|\ + tplink,ex447) + ucidef_set_interface_wan "eth0" + ucidef_set_interface_lan "" + ;; + qcom,ipq6018-cp01|\ + qcom,ipq807x-hk01) + ucidef_set_interface_lan "eth0 eth1 eth2 eth3 eth4" + ucidef_set_interface_wan "eth5" + ;; + cig,wf194c|\ + cig,wf194c4|\ + edgecore,eap106|\ + qcom,ipq5018-mp03.3|\ + sercomm,wallaby) + ucidef_set_interface_lan "eth0" + ucidef_set_interface_wan "eth1" + ;; + edgecore,eap101) + ucidef_set_interface_lan "eth1 eth2" + ucidef_set_interface_wan "eth0" + ;; + edgecore,eap102|\ + wallys,dr6018|\ + cig,wf188n|\ + cig,wf196) + ucidef_set_interface_lan "eth1" + ucidef_set_interface_wan "eth0" + ;; + qcom,ipq807x-hk14) + ucidef_set_interface_lan "eth0 eth1 eth2 eth3" + ucidef_set_interface_wan "eth4" + ;; + wallys,dr6018-v4) + ucidef_set_interface_lan "eth1 eth2 eth3 eth4" + ucidef_set_interface_wan "eth0" + ;; + esac +} + +qcom_setup_macs() +{ + local board="$1" + + case $board in + cig,wf194c|\ + cig,wf194c4) + mac=$(grep BaseMacAddress= /dev/mtd14 | cut -dx -f2) + wan_mac=$(macaddr_canonicalize $mac) + lan_mac=$(macaddr_add "$wan_mac" 1) + ucidef_set_network_device_mac eth0 $lan_mac + ucidef_set_network_device_mac eth1 $wan_mac + ucidef_set_label_macaddr $wan_mac + ;; + *) + wan_mac=$(cat /sys/class/net/eth0/address) + lan_mac=$(macaddr_add "$wan_mac" 1) + ;; + esac + [ -n "$lan_mac" ] && ucidef_set_interface_macaddr "lan" $lan_mac + [ -n "$wan_mac" ] && ucidef_set_interface_macaddr "wan" $wan_mac +} + +board_config_update +board=$(board_name) +ucidef_set_bridge_device bridge +qcom_setup_interfaces $board +qcom_setup_macs $board +board_config_flush + +exit 0 diff --git a/feeds/ipq807x/ipq807x/base-files/etc/board.d/03_wifi b/feeds/ipq807x/ipq807x/base-files/etc/board.d/03_wifi new file mode 100755 index 000000000..200e43d15 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/board.d/03_wifi @@ -0,0 +1,17 @@ +#!/bin/sh + +. /lib/functions/uci-defaults.sh + +board_config_update + +board=$(board_name) + +case "$board" in +edgecore,eap106) + ucidef_set_wifi_scanning 'soc/10000000.pci/pci0001:00/0001:00:00.0/0001:01:00.0' + ;; +esac + +board_config_flush + +exit 0 diff --git a/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata new file mode 100755 index 000000000..327ba8546 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata @@ -0,0 +1,116 @@ +#!/bin/sh + +[ -e /lib/firmware/$FIRMWARE ] && exit 0 + +. /lib/functions.sh +. /lib/functions/system.sh + +ath11k_generate_macs() { + touch /lib/firmware/ath11k-macs + eth=$(cat /sys/class/net/eth0/address) + mac1=$(macaddr_add $eth 2) + mac2=$(macaddr_add $eth 10) + mac3=$(macaddr_add $eth 18) + echo -ne \\x${mac1//:/\\x} >> /lib/firmware/ath11k-macs + echo -ne \\x${mac2//:/\\x} >> /lib/firmware/ath11k-macs + echo -ne \\x${mac3//:/\\x} >> /lib/firmware/ath11k-macs +} + +ath11k_generate_macs_wf194() { + touch /lib/firmware/ath11k-macs + mac=$(grep BaseMacAddress= /dev/mtd14 | cut -dx -f2) + eth=$(macaddr_canonicalize $mac) + mac1=$(macaddr_add $eth 2) + mac2=$(macaddr_add $eth 10) + mac3=$(macaddr_add $eth 18) + echo -ne \\x${mac1//:/\\x} >> /lib/firmware/ath11k-macs + echo -ne \\x${mac2//:/\\x} >> /lib/firmware/ath11k-macs + echo -ne \\x${mac3//:/\\x} >> /lib/firmware/ath11k-macs +} + +caldata_die() { + echo "caldata: " "$*" + exit 1 +} + +caldata_extract() { + local part=$1 + local offset=$(($2)) + local count=$(($3)) + local mtd + + mtd=$(find_mtd_chardev $part) + [ -n "$mtd" ] || caldata_die "no mtd device found for partition $part" + + dd if=$mtd of=/lib/firmware/$FIRMWARE iflag=skip_bytes bs=$count skip=$offset count=1 2>/dev/null || \ + caldata_die "failed to extract calibration data from $mtd" +} + +board=$(board_name) + +case "$FIRMWARE" in +"ath11k/IPQ8074/hw2.0/caldata.bin") + case "$board" in + cig,wf194c|\ + cig,wf194c4|\ + cig,wf196|\ + edgecore,eap102 |\ + edgecore,eap106 |\ + qcom,ipq807x-hk01|\ + qcom,ipq807x-hk14|\ + tplink,ex227|\ + tplink,ex447|\ + sercomm,wallaby) + caldata_extract "0:ART" 0x1000 0x20000 + ;; + esac + ;; +"ath11k/IPQ6018/hw1.0/caldata.bin") + case "$board" in + cig,wf188|\ + cig,wf188n|\ + edgecore,eap101|\ + hfcl,ion4xi|\ + hfcl,ion4xe|\ + wallys,dr6018|\ + wallys,dr6018-v4|\ + qcom,ipq6018-cp01|\ + xiaomi,ax1800) + caldata_extract "0:ART" 0x1000 0x20000 + ;; + esac + ;; +ath11k/QCN9074/hw1.0/caldata_1.bin) + case "$board" in + cig,wf196|\ + qcom,ipq807x-hk14) + caldata_extract "0:ART" 0x26800 0x20000 + ;; + esac + ;; +ath11k/QCN9074/hw1.0/caldata_2.bin) + case "$board" in + qcom,ipq807x-hk14|\ + qcom,ipq5018-mp03.3) + caldata_extract "0:ART" 0x4C000 0x20000 + ;; + esac + ;; +ath11k-macs) + case "$board" in + edgecore,eap101|\ + edgecore,eap102|\ + edgecore,eap106|\ + cig,wf188n) + ath11k_generate_macs + ;; + cig,wf194c|\ + cig,wf194c) + ath11k_generate_macs_wf194 + ;; + esac + ;; +*) + exit 1 + ;; +esac diff --git a/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata new file mode 100644 index 000000000..eb23e8cd7 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata @@ -0,0 +1,16 @@ +#!/bin/sh + +[ -e /lib/firmware/$FIRMWARE ] && exit 0 + +. /lib/functions/caldata.sh + +board=$(board_name) + +case "$FIRMWARE" in +"ath10k/pre-cal-pci-0001:01:00.0.bin") + case $board in + edgecore,eap106) + caldata_extract "0:ART" 0x33000 0x2f20 + ;; + esac +esac diff --git a/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/net/macs b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/net/macs new file mode 100644 index 000000000..10f38108c --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/hotplug.d/net/macs @@ -0,0 +1,3 @@ +#!/bin/sh +mac=$(cat /etc/board.json | jsonfilter -e '@["network-device"]["'$DEVICENAME'"]'.macaddr) +[ -n "$mac" ] && ip link set $DEVICENAME address $mac diff --git a/feeds/ipq807x/ipq807x/base-files/etc/init.d/aq_phy b/feeds/ipq807x/ipq807x/base-files/etc/init.d/aq_phy new file mode 100755 index 000000000..e4841e189 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/init.d/aq_phy @@ -0,0 +1,22 @@ +#!/bin/sh /etc/rc.common + +START=30 + +boot() { + . /lib/functions.sh + + case "$(board_name)" in + cig,wf194c|\ + cig,wf194c4|\ + cig,wf196) + aq-fw-download /lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld miireg 0 > /dev/null + sleep 1 + ssdk_sh debug phy set 0 0x4004c441 0x8 + ;; + edgecore,eap106) + aq-fw-download /lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld miireg 8 > /dev/null + sleep 1 + ssdk_sh debug phy set 8 0x4004c441 0x8 + ;; + esac +} diff --git a/feeds/ipq807x/ipq807x/base-files/etc/init.d/bootcount b/feeds/ipq807x/ipq807x/base-files/etc/init.d/bootcount new file mode 100755 index 000000000..a24f27353 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/init.d/bootcount @@ -0,0 +1,12 @@ +#!/bin/sh /etc/rc.common + +START=99 + +boot() { + case "$(board_name)" in + edgecore,eap101|\ + edgecore,eap102) + fw_setenv bootcount 0 + ;; + esac +} diff --git a/feeds/ipq807x/ipq807x/base-files/etc/init.d/wdt b/feeds/ipq807x/ipq807x/base-files/etc/init.d/wdt new file mode 100755 index 000000000..75de36a34 --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/etc/init.d/wdt @@ -0,0 +1,14 @@ +#!/bin/sh /etc/rc.common + +START=10 + +boot() { + . /lib/functions.sh + + case "$(board_name)" in + cig,wf188|\ + cig,wf188n) + watchdog -t 5 /dev/watchdog1 + ;; + esac +} diff --git a/feeds/ipq807x/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld b/feeds/ipq807x/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld new file mode 100644 index 000000000..4a5a76cd3 Binary files /dev/null and b/feeds/ipq807x/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld differ diff --git a/feeds/ipq807x/ipq807x/base-files/lib/upgrade/platform.sh b/feeds/ipq807x/ipq807x/base-files/lib/upgrade/platform.sh new file mode 100755 index 000000000..1cd40b29f --- /dev/null +++ b/feeds/ipq807x/ipq807x/base-files/lib/upgrade/platform.sh @@ -0,0 +1,79 @@ +. /lib/functions/system.sh + +qca_do_upgrade() { + local tar_file="$1" + + local board_dir=$(tar tf $tar_file | grep -m 1 '^sysupgrade-.*/$') + board_dir=${board_dir%/} + local dev=$(find_mtd_chardev "0:HLOS") + + tar Oxf $tar_file ${board_dir}/kernel | mtd write - ${dev} + + if [ -n "$UPGRADE_BACKUP" ]; then + tar Oxf $tar_file ${board_dir}/root | mtd -j "$UPGRADE_BACKUP" write - rootfs + else + tar Oxf $tar_file ${board_dir}/root | mtd write - rootfs + fi +} + +platform_check_image() { + local magic_long="$(get_magic_long "$1")" + board=$(board_name) + case $board in + cig,wf188|\ + cig,wf188n|\ + cig,wf194c|\ + cig,wf194c4|\ + wallys,dr6018|\ + wallys,dr6018-v4|\ + edgecore,eap101|\ + edgecore,eap102|\ + edgecore,eap106|\ + hfcl,ion4xi|\ + hfcl,ion4xe|\ + tplink,ex227|\ + tplink,ex447|\ + qcom,ipq6018-cp01|\ + qcom,ipq807x-hk01|\ + qcom,ipq807x-hk14|\ + qcom,ipq5018-mp03.3) + [ "$magic_long" = "73797375" ] && return 0 + ;; + esac + return 1 +} + +platform_do_upgrade() { + CI_UBIPART="rootfs" + CI_ROOTPART="ubi_rootfs" + CI_IPQ807X=1 + + board=$(board_name) + case $board in + cig,wf188) + qca_do_upgrade $1 + ;; + cig,wf188n|\ + cig,wf194c|\ + cig,wf194c4|\ + hfcl,ion4xi|\ + hfcl,ion4xe|\ + qcom,ipq6018-cp01|\ + qcom,ipq807x-hk01|\ + qcom,ipq807x-hk14|\ + qcom,ipq5018-mp03.3|\ + wallys,dr6018|\ + wallys,dr6018-v4|\ + tplink,ex447|\ + tplink,ex227) + nand_upgrade_tar "$1" + ;; + edgecore,eap106|\ + edgecore,eap102|\ + edgecore,eap101) + CI_UBIPART="rootfs1" + [ "$(find_mtd_chardev rootfs)" ] && CI_UBIPART="rootfs" + nand_upgrade_tar "$1" + ;; + esac +} diff --git a/feeds/ipq807x/ipq807x/config-4.4 b/feeds/ipq807x/ipq807x/config-4.4 new file mode 100644 index 000000000..af9953aea --- /dev/null +++ b/feeds/ipq807x/ipq807x/config-4.4 @@ -0,0 +1,829 @@ +# CONFIG_AHCI_IPQ is not set +CONFIG_ALIGNMENT_TRAP=y +# CONFIG_ALLOW_DEV_COREDUMP is not set +# CONFIG_AMBA_PL08X is not set +# CONFIG_APM_EMULATION is not set +# CONFIG_APQ_GCC_8084 is not set +# CONFIG_APQ_MMCC_8084 is not set +# CONFIG_AR8216_PHY is not set +CONFIG_ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE=y +CONFIG_ARCH_HAS_ELF_RANDOMIZE=y +CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y +CONFIG_ARCH_HAS_SG_CHAIN=y +CONFIG_ARCH_HAS_TICK_BROADCAST=y +CONFIG_ARCH_HAVE_CUSTOM_GPIO_H=y +# CONFIG_ARCH_IPQ40XX is not set +# CONFIG_ARCH_IPQ806x is not set +# CONFIG_ARCH_IPQ807x is not set +# CONFIG_ARCH_IPQ6018 is not set +# CONFIG_ARCH_IPQ5018 is not set +CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y +# CONFIG_ARCH_MSM8960 is not set +# CONFIG_ARCH_MSM8974 is not set +CONFIG_ARCH_MSM8X60=y +CONFIG_ARCH_MULTIPLATFORM=y +# CONFIG_ARCH_MULTI_CPU_AUTO is not set +CONFIG_ARCH_MULTI_V6_V7=y +CONFIG_ARCH_MULTI_V7=y +CONFIG_ARCH_NR_GPIO=0 +CONFIG_ARCH_QCOM=y +CONFIG_QSEECOM=m +# CONFIG_ARCH_SELECT_MEMORY_MODEL is not set +# CONFIG_ARCH_SPARSEMEM_DEFAULT is not set +CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=y +CONFIG_ARCH_SUPPORTS_BIG_ENDIAN=y +CONFIG_ARCH_SUPPORTS_UPROBES=y +CONFIG_ARCH_SUSPEND_POSSIBLE=y +CONFIG_ARCH_USE_BUILTIN_BSWAP=y +CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y +CONFIG_ARCH_WANT_GENERAL_HUGETLB=y +CONFIG_ARCH_WANT_IPC_PARSE_VERSION=y +CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y +CONFIG_ARM=y +CONFIG_ARM_AMBA=y +CONFIG_ARM_ARCH_TIMER=y +CONFIG_ARM_ARCH_TIMER_EVTSTREAM=y +# CONFIG_ARM_ATAG_DTB_COMPAT is not set +CONFIG_ARM_CCI=y +CONFIG_ARM_CCI400_COMMON=y +CONFIG_ARM_CCI400_PMU=y +CONFIG_ARM_CCI_PMU=y +CONFIG_ARM_CPU_SUSPEND=y +CONFIG_ARM_GIC=y +CONFIG_ARM_HAS_SG_CHAIN=y +# CONFIG_ARM_HIGHBANK_CPUIDLE is not set +CONFIG_ARM_CPUIDLE=y +CONFIG_ARM_L1_CACHE_SHIFT=6 +CONFIG_ARM_L1_CACHE_SHIFT_6=y +# CONFIG_ARM_LPAE is not set +CONFIG_ARM_MODULE_PLTS=y +CONFIG_ARM_PATCH_PHYS_VIRT=y +CONFIG_ARM_PMU=y +CONFIG_ARM_PSCI=y +CONFIG_ARM_PSCI_FW=y +CONFIG_ARM_QCOM_CPUFREQ=y +# CONFIG_ARM_SMMU is not set +# CONFIG_ARM_SP805_WATCHDOG is not set +CONFIG_ARM_THUMB=y +# CONFIG_ARM_THUMBEE is not set +CONFIG_ARM_UNWIND=y +CONFIG_ARM_VIRT_EXT=y +CONFIG_AT803X_PHY=y +# CONFIG_ATA is not set +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NVME=y +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_COUNT=16 +CONFIG_BLK_DEV_RAM_SIZE=4096 +# CONFIG_VIRTIO_BLK is not set +# CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set +CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0 +CONFIG_BOUNCE=y +CONFIG_BUILD_BIN2C=y +# CONFIG_CNSS_QCN9000 is not set +# CONFIG_CNSS2 is not set +# CONFIG_CNSS2_GENL is not set +# CONFIG_CNSS2_DEBUG is not set +# CONFIG_CNSS2_PM is not set +# CONFIG_CNSS2_PCI_DRIVER is not set +# CONFIG_CNSS2_CALIBRATION_SUPPORT is not set +# CONFIG_CNSS2_SMMU is not set +# CONFIG_CNSS2_RAMDUMP is not set +# CONFIG_CACHE_L2X0 is not set +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +CONFIG_CC_STACKPROTECTOR=y +# CONFIG_CC_STACKPROTECTOR_NONE is not set +CONFIG_CC_STACKPROTECTOR_REGULAR=y +# CONFIG_CHARGER_QCOM_SMBB is not set +CONFIG_CLEANCACHE=y +CONFIG_CLKDEV_LOOKUP=y +CONFIG_CLKSRC_OF=y +CONFIG_CLKSRC_PROBE=y +CONFIG_CLKSRC_QCOM=y +CONFIG_CLONE_BACKWARDS=y +CONFIG_COMMON_CLK=y +CONFIG_COMMON_CLK_QCOM=y +CONFIG_CONFIGFS_FS=y +CONFIG_COREDUMP=y +# CONFIG_CORESIGHT is not set +# CONFIG_CORESIGHT_CSR is not set +# CONFIG_CORESIGHT_CTI is not set +# NFIG_CORESIGHT_EVENT is not set +# CONFIG_CORESIGHT_HWEVENT is not set +# CONFIG_CORESIGHT_LINKS_AND_SINKS is not set +# CONFIG_CORESIGHT_LINK_AND_SINK_TMC is not set +# CONFIG_CORESIGHT_QCOM_REPLICATOR is not set +# CONFIG_CORESIGHT_QPDI is not set +# CONFIG_CORESIGHT_SINK_ETBV10 is not set +# CONFIG_CORESIGHT_SINK_TPIU is not set +# CONFIG_CORESIGHT_SOURCE_DUMMY is not set +# CONFIG_CORESIGHT_SOURCE_ETM3X is not set +# CONFIG_CORESIGHT_SOURCE_ETM4X is not set +# CONFIG_CORESIGHT_REMOTE_ETM is not set +# CONFIG_CORESIGHT_STM is not set +# CONFIG_CORESIGHT_TPDA is not set +# CONFIG_CORESIGHT_TPDM is not set +# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set +# CONFIG_CORESIGHT_STREAM is not set +CONFIG_CPUFREQ_DT=y +CONFIG_CPUFREQ_DT_PLATDEV=y +CONFIG_CPU_32v6K=y +CONFIG_CPU_32v7=y +CONFIG_CPU_ABRT_EV7=y +# CONFIG_CPU_BIG_ENDIAN is not set +# CONFIG_CPU_BPREDICT_DISABLE is not set +CONFIG_CPU_CACHE_V7=y +CONFIG_CPU_CACHE_VIPT=y +CONFIG_CPU_COPY_V6=y +CONFIG_CPU_CP15=y +CONFIG_CPU_CP15_MMU=y +# CONFIG_CPU_SW_DOMAIN_PAN is not set +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_COMMON=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_HAS_ASID=y +# CONFIG_CPU_ICACHE_DISABLE is not set +CONFIG_CPU_IDLE=y +CONFIG_CPU_IDLE_GOV_LADDER=y +CONFIG_CPU_IDLE_GOV_MENU=y +CONFIG_CPU_PABRT_V7=y +CONFIG_CPU_PM=y +CONFIG_CPU_RMAP=y +# CONFIG_CPU_THERMAL is not set +CONFIG_CPU_TLB_V7=y +CONFIG_CPU_V7=y +CONFIG_CRC16=y +# CONFIG_CRC32_SARWATE is not set +CONFIG_CRC32_SLICEBY8=y +CONFIG_CROSS_MEMORY_ATTACH=y +# CONFIG_CRYPTO_DEV_QCOM_MSM_QCE is not set +# CONFIG_CRYPTO_DEV_OTA_CRYPTO is not set +# CONFIG_FIPS_ENABLE is not set +CONFIG_CRYPTO_AEAD=y +CONFIG_CRYPTO_AEAD2=y +CONFIG_CRYPTO_BLKCIPHER=y +CONFIG_CRYPTO_BLKCIPHER2=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_CRC32C=y +CONFIG_CRYPTO_CTR=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_CMAC=y +# CONFIG_CRYPTO_DEV_QCOM_ICE is not set +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_ECB=y +CONFIG_CRYPTO_HASH2=y +CONFIG_CRYPTO_HASH_INFO=y +CONFIG_CRYPTO_HW=y +CONFIG_CRYPTO_LZO=y +CONFIG_CRYPTO_MANAGER=y +CONFIG_CRYPTO_MANAGER2=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_RNG=y +CONFIG_CRYPTO_RNG2=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_WORKQUEUE=y +CONFIG_CRYPTO_XZ=y +CONFIG_CRYPTO_ARC4=y +CONFIG_CRYPTO_GCM=y +CONFIG_DCACHE_WORD_ACCESS=y +CONFIG_DEBUG_BUGVERBOSE=y +CONFIG_DEBUG_GPIO=y +# CONFIG_DEBUG_INFO_REDUCED is not set +CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" +# CONFIG_DEBUG_MEM_USAGE is not set +# CONFIG_DEBUG_UART_8250 is not set +# CONFIG_DEBUG_USER is not set +CONFIG_DECOMPRESS_GZIP=y +CONFIG_DEVMEM=y +# CONFIG_DIAG_OVER_USB is not set +CONFIG_DMADEVICES=y +CONFIG_DMA_ENGINE=y +CONFIG_DMA_OF=y +CONFIG_DMA_VIRTUAL_CHANNELS=y +CONFIG_DTC=y +# CONFIG_DWMAC_GENERIC is not set +# CONFIG_DWMAC_IPQ806X is not set +# CONFIG_DWMAC_SUNXI is not set +# CONFIG_DW_DMAC_PCI is not set +# CONFIG_VHOST_NET is not set +# CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set +CONFIG_DYNAMIC_DEBUG=y +CONFIG_ETHERNET_PACKET_MANGLE=y +CONFIG_EXT4_FS=y +# CONFIG_EXT4_USE_FOR_EXT2 is not set +CONFIG_FB=y +CONFIG_FB_CFB_COPYAREA=y +CONFIG_FB_CFB_FILLRECT=y +CONFIG_FB_CFB_IMAGEBLIT=y +CONFIG_FB_CMDLINE=y +CONFIG_FB_QCOM_QPIC=y +CONFIG_FB_QCOM_QPIC_ER_SSD1963_PANEL=y +CONFIG_FB_SYS_FOPS=y +CONFIG_FIXED_PHY=y +CONFIG_FIX_EARLYCON_MEM=y +CONFIG_FS_MBCACHE=y +CONFIG_GENERIC_ALLOCATOR=y +CONFIG_GENERIC_BUG=y +CONFIG_GENERIC_CLOCKEVENTS=y +CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y +# CONFIG_GENERIC_CPUFREQ_KRAIT is not set +CONFIG_GENERIC_IDLE_POLL_SETUP=y +CONFIG_GENERIC_IO=y +CONFIG_GENERIC_IRQ_SHOW=y +CONFIG_GENERIC_IRQ_SHOW_LEVEL=y +CONFIG_GENERIC_MSI_IRQ=y +CONFIG_GENERIC_PCI_IOMAP=y +CONFIG_GENERIC_PHY=y +CONFIG_GENERIC_PINCONF=y +CONFIG_GENERIC_SCHED_CLOCK=y +CONFIG_GENERIC_SMP_IDLE_THREAD=y +CONFIG_GENERIC_STRNCPY_FROM_USER=y +CONFIG_GENERIC_STRNLEN_USER=y +CONFIG_GENERIC_TIME_VSYSCALL=y +CONFIG_GPIOLIB=y +CONFIG_GPIOLIB_IRQCHIP=y +CONFIG_GPIO_DEVRES=y +# CONFIG_GPIO_LATCH is not set +# CONFIG_GPIO_NXP_74HC153 is not set +CONFIG_GPIO_SYSFS=y +CONFIG_HANDLE_DOMAIN_IRQ=y +CONFIG_HARDIRQS_SW_RESEND=y +CONFIG_HAS_DMA=y +CONFIG_HAS_IOMEM=y +CONFIG_HAS_IOPORT_MAP=y +# CONFIG_HAVE_64BIT_ALIGNED_ACCESS is not set +CONFIG_HAVE_ARCH_AUDITSYSCALL=y +CONFIG_HAVE_ARCH_BITREVERSE=y +CONFIG_HAVE_ARCH_JUMP_LABEL=y +CONFIG_HAVE_ARCH_KGDB=y +CONFIG_HAVE_ARCH_PFN_VALID=y +CONFIG_HAVE_ARCH_SECCOMP_FILTER=y +CONFIG_HAVE_ARCH_TRACEHOOK=y +CONFIG_HAVE_ARM_ARCH_TIMER=y +# CONFIG_HAVE_BOOTMEM_INFO_NODE is not set +CONFIG_HAVE_BPF_JIT=y +CONFIG_HAVE_CC_STACKPROTECTOR=y +CONFIG_HAVE_CLK=y +CONFIG_HAVE_CLK_PREPARE=y +CONFIG_HAVE_CONTEXT_TRACKING=y +CONFIG_HAVE_C_RECORDMCOUNT=y +CONFIG_HAVE_DEBUG_KMEMLEAK=y +CONFIG_HAVE_DMA_API_DEBUG=y +CONFIG_HAVE_DMA_ATTRS=y +CONFIG_HAVE_DMA_CONTIGUOUS=y +CONFIG_HAVE_DYNAMIC_FTRACE=y +CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y +CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y +CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y +CONFIG_HAVE_FUNCTION_TRACER=y +# CONFIG_SRD_TRACE is not set +CONFIG_HAVE_GENERIC_DMA_COHERENT=y +CONFIG_HAVE_HW_BREAKPOINT=y +CONFIG_HAVE_IDE=y +CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y +CONFIG_HAVE_KERNEL_GZIP=y +CONFIG_HAVE_KERNEL_LZ4=y +CONFIG_HAVE_KERNEL_LZMA=y +CONFIG_HAVE_KERNEL_LZO=y +CONFIG_HAVE_KERNEL_XZ=y +# CONFIG_HAVE_KPROBES is not set +# CONFIG_HAVE_KRETPROBES is not set +CONFIG_HAVE_MEMBLOCK=y +CONFIG_HAVE_MOD_ARCH_SPECIFIC=y +CONFIG_HAVE_NET_DSA=y +CONFIG_HAVE_OPROFILE=y +# CONFIG_HAVE_OPTPROBES is not set +CONFIG_HAVE_PERF_EVENTS=y +CONFIG_HAVE_PERF_REGS=y +CONFIG_HAVE_PERF_USER_STACK_DUMP=y +CONFIG_HAVE_PROC_CPU=y +CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y +CONFIG_HAVE_SMP=y +CONFIG_HAVE_SYSCALL_TRACEPOINTS=y +CONFIG_HAVE_UID16=y +CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y +CONFIG_HIGHMEM=y +CONFIG_HIGHPTE=y +CONFIG_HOTPLUG_CPU=y +CONFIG_HWMON=y +CONFIG_HWSPINLOCK=y +CONFIG_HWSPINLOCK_QCOM=y +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_MSM=y +CONFIG_HZ_FIXED=0 +CONFIG_I2C=y +CONFIG_I2C_BOARDINFO=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_COMPAT=y +CONFIG_I2C_HELPER_AUTO=y +CONFIG_I2C_QUP=y +# CONFIG_IIO is not set +# CONFIG_IIO_BUFFER is not set +# CONFIG_IIO_TRIGGER is not set +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_INITRAMFS_SOURCE="" +# CONFIG_INPUT_PM8941_PWRKEY is not set +CONFIG_IOMMU_HELPER=y +# CONFIG_IOMMU_IO_PGTABLE_LPAE is not set +# CONFIG_IPQ_DWC3_QTI_EXTCON is not set +# CONFIG_IPQ_GCC_4019 is not set +# CONFIG_IPQ_GCC_5018 is not set +# CONFIG_IPQ_APSS_5018 is not set +# CONFIG_IPQ_GCC_6018 is not set +# CONFIG_IPQ_APSS_6018 is not set +# CONFIG_IPQ_GCC_806X is not set +# CONFIG_IPQ_ADSS_807x is not set +# CONFIG_IPQ_APSS_807x is not set +# CONFIG_IPQ_GCC_807x is not set +# CONFIG_IPQ_ADCC_4019 is not set +# CONFIG_IPQ_LCC_806X is not set +# CONFIG_IPQ_REMOTEPROC_ADSP is not set +# CONFIG_IPQ_SUBSYSTEM_RESTART is not set +# CONFIG_IPQ_SUBSYSTEM_RESTART_TEST is not set +CONFIG_IRQCHIP=y +CONFIG_IRQ_DOMAIN=y +CONFIG_IRQ_DOMAIN_HIERARCHY=y +CONFIG_IRQ_FORCED_THREADING=y +CONFIG_IRQ_WORK=y +CONFIG_JBD2=y +# CONFIG_IPC_ROUTER is not set +# CONFIG_IPC_ROUTER_SECURITY is not set +# CONFIG_IPC_LOGGING is not set +CONFIG_KPSS_XCC=y +# CONFIG_KRAITCC is not set +# CONFIG_KRAIT_CLOCKS is not set +# CONFIG_KRAIT_L2_ACCESSORS is not set +CONFIG_LEDS_IPQ=y +CONFIG_LEDS_PWM=y +CONFIG_LEDS_TLC591XX=y +# CONFIG_LEDS_PCA9956B is not set +CONFIG_LIBFDT=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_LOCK_SPIN_ON_OWNER=y +CONFIG_LZO_COMPRESS=y +CONFIG_LZO_DECOMPRESS=y +CONFIG_MDIO=y +CONFIG_MDIO_BITBANG=y +CONFIG_MDIO_BOARDINFO=y +CONFIG_MDIO_GPIO=y +# CONFIG_MDIO_QCA is not set +CONFIG_MFD_QCOM_RPM=y +CONFIG_MFD_SPMI_PMIC=y +# CONFIG_SLIMBUS is not set +# CONFIG_SLIMBUS_MSM_CTRL is not set +# CONFIG_SLIMBUS_MSM_NGD is not set +# CONFIG_OF_SLIMBUS is not set +CONFIG_MFD_SYSCON=y +CONFIG_MIGHT_HAVE_CACHE_L2X0=y +CONFIG_MIGHT_HAVE_PCI=y +CONFIG_MMC=y +CONFIG_MMC_ARMMMCI=y +CONFIG_MMC_BLOCK=y +CONFIG_MMC_BLOCK_MINORS=32 +CONFIG_MMC_QCOM_DML=y +CONFIG_MMC_QCOM_TUNING=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_MSM=y +# CONFIG_MMC_SDHCI_OF_ARASAN is not set +# CONFIG_MMC_SDHCI_PCI is not set +CONFIG_MMC_SDHCI_PLTFM=y +# CONFIG_MMC_TIFM_SD is not set +CONFIG_MODULES_TREE_LOOKUP=y +CONFIG_MODULES_USE_ELF_REL=y +# CONFIG_MPLS_ROUTING is not set +# CONFIG_MSM_GCC_8660 is not set +# CONFIG_MSM_GCC_8916 is not set +# CONFIG_MSM_GCC_8960 is not set +# CONFIG_MSM_GCC_8974 is not set +# CONFIG_MSM_LCC_8960 is not set +# CONFIG_MSM_MMCC_8960 is not set +# CONFIG_MSM_MMCC_8974 is not set +# CONFIG_MSM_MHI is not set +# CONFIG_MSM_IPC_ROUTER_MHI_XPRT is not set +# CONFIG_MSM_MHI_DEBUG is not set +# CONFIG_MSM_MHI_DEV is not set +# CONFIG_MSM_MHI_UCI is not set +# CONFIG_DIAGFWD_BRIDGE_CODE is not set +# CONFIG_MSM_BUS_SCALING is not set +# CONFIG_BUS_TOPOLOGY_ADHOC is not set +# CONFIG_QPNP_REVID is not set +# CONFIG_SPS is not set +# CONFIG_SPS_SUPPORT_NDP_BAM is not set +# CONFIG_USB_BAM is not set +# CONFIG_SPS_SUPPORT_BAMDMA is not set +# CONFIG_IPA is not set +# CONFIG_IPA3 is not set +# CONFIG_EP_PCIE is not set +# CONFIG_GSI is not set +# CONFIG_PFT is not set +# CONFIG_SEEMP_CORE is not set +# CONFIG_GPIO_USB_DETECT is not set +# CONFIG_MSM_GLINK is not set +# CONFIG_MSM_GLINK_LOOPBACK_SERVER is not set +# CONFIG_MSM_GLINK_SMEM_NATIVE_XPRT is not set +# CONFIG_MSM_GLINK_PKT is not set +# CONFIG_MSM_IPC_ROUTER_GLINK_XPRT is not set +# CONFIG_MSM_QMI_INTERFACE is not set +# CONFIG_MSM_TEST_QMI_CLIENT is not set +# CONFIG_GLINK_DEBUG_FS is not set +# CONFIG_MSM_RPM_SMD is not set +# CONFIG_MSM_RPM_GLINK is not set +CONFIG_MSM_RPM_LOG=y +# CONFIG_MSM_SMEM is not set +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_M25P80=y +CONFIG_MTD_NAND=y +CONFIG_MTD_NAND_ECC=y +CONFIG_MTD_NAND_QCOM=y +CONFIG_MTD_QCOM_SMEM_PARTS=y +CONFIG_MTD_SPINAND_GIGADEVICE=y +CONFIG_MTD_SPINAND_MT29F=y +CONFIG_MTD_SPINAND_ONDIEECC=y +CONFIG_MTD_SPI_NOR=y +CONFIG_MTD_SPLIT_FIRMWARE=y +CONFIG_MTD_SPLIT_FIT_FW=y +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_BEB_LIMIT=20 +CONFIG_MTD_UBI_BLOCK=y +# CONFIG_MTD_UBI_FASTMAP is not set +CONFIG_MTD_UBI_GLUEBI=y +CONFIG_MTD_UBI_WL_THRESHOLD=4096 +CONFIG_MULTI_IRQ_HANDLER=y +CONFIG_MUTEX_SPIN_ON_OWNER=y +CONFIG_NEED_DMA_MAP_STATE=y +CONFIG_NEON=y +CONFIG_NET=y +# CONFIG_NET_DSA_MV88E6063 is not set +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NET_PTP_CLASSIFY=y +CONFIG_NO_BOOTMEM=y +CONFIG_NO_HZ=y +CONFIG_NO_HZ_COMMON=y +CONFIG_NO_HZ_IDLE=y +CONFIG_NR_CPUS=4 +CONFIG_NUM_ALT_PARTITION=8 +CONFIG_NVMEM=y +CONFIG_OF=y +CONFIG_OF_ADDRESS=y +CONFIG_OF_ADDRESS_PCI=y +CONFIG_OF_EARLY_FLATTREE=y +CONFIG_OF_FLATTREE=y +CONFIG_OF_GPIO=y +CONFIG_OF_IRQ=y +CONFIG_OF_MDIO=y +CONFIG_OF_MTD=y +CONFIG_OF_NET=y +CONFIG_OF_PCI=y +CONFIG_OF_PCI_IRQ=y +CONFIG_OF_RESERVED_MEM=y +CONFIG_OLD_SIGACTION=y +CONFIG_OLD_SIGSUSPEND3=y +CONFIG_PAGE_OFFSET=0xC0000000 +CONFIG_PANIC_ON_OOPS=y +CONFIG_PANIC_ON_OOPS_VALUE=1 +CONFIG_PANIC_TIMEOUT=5 +CONFIG_PCI=y +# CONFIG_PCIEAER is not set +CONFIG_PCIE_DW=y +# CONFIG_PCIE_DW_PLAT is not set +CONFIG_PCIE_PME=y +CONFIG_PCIE_QCOM=y +CONFIG_PCI_DOMAINS=y +CONFIG_PCI_DOMAINS_GENERIC=y +CONFIG_PCI_MSI=y +CONFIG_PERF_EVENTS=y +CONFIG_PERF_USE_VMALLOC=y +CONFIG_PGTABLE_LEVELS=2 +CONFIG_PHYLIB=y +# CONFIG_PHY_IPQ_BALDUR_USB is not set +# CONFIG_PHY_IPQ_UNIPHY_USB is not set +# CONFIG_PHY_QCOM_APQ8064_SATA is not set +# CONFIG_PHY_QCOM_IPQ806X_SATA is not set +CONFIG_PHY_QCA_PCIE_QMP=y +# CONFIG_PHY_QCOM_UFS is not set +# CONFIG_PHY_IPQ_UNIPHY_PCIE is not set +CONFIG_PINCTRL=y +# CONFIG_PINCTRL_APQ8064 is not set +# CONFIG_PINCTRL_APQ8084 is not set +# CONFIG_PINCTRL_IPQ4019 is not set +# CONFIG_PINCTRL_IPQ6018 is not set +# CONFIG_PINCTRL_IPQ8064 is not set +# CONFIG_PINCTRL_IPQ807x is not set +# CONFIG_PINCTRL_IPQ5018 is not set +CONFIG_PINCTRL_MSM=y +# CONFIG_PINCTRL_MSM8660 is not set +# CONFIG_PINCTRL_MSM8916 is not set +# CONFIG_PINCTRL_MSM8960 is not set +CONFIG_PINCTRL_QCOM_SPMI_PMIC=y +# CONFIG_PINCTRL_QCOM_SSBI_PMIC is not set +# CONFIG_PL330_DMA is not set +CONFIG_PM=y +CONFIG_PM_CLK=y +# CONFIG_PM_DEBUG is not set +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_PM_GENERIC_DOMAINS_OF=y +CONFIG_PM_GENERIC_DOMAINS_SLEEP=y +CONFIG_PM_OPP=y +CONFIG_PM_SLEEP=y +CONFIG_PM_SLEEP_SMP=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_MSM=y +CONFIG_POWER_SUPPLY=y +CONFIG_PREEMPT=y +CONFIG_PREEMPT_COUNT=y +# CONFIG_PREEMPT_NONE is not set +CONFIG_PREEMPT_RCU=y +CONFIG_PRINTK_TIME=y +CONFIG_PROC_PAGE_MONITOR=y +# CONFIG_PROC_STRIPPED is not set +CONFIG_PSTORE=y +CONFIG_PSTORE_RAM=y +# CONFIG_PSTORE_CONSOLE is not set +# CONFIG_PSTORE_PMSG is not set +CONFIG_PWM=y +CONFIG_PWM_IPQ4019=y +# CONFIG_PWM_PCA9685 is not set +CONFIG_PWM_SYSFS=y +CONFIG_QCOM_ADM=y +# CONFIG_QCOM_APM is not set +CONFIG_QCOM_BAM_DMA=y +# CONFIG_QTI_BT_TTY is not set +# CONFIG_QCOM_COINCELL is not set +# CONFIG_QCOM_DCC is not set +CONFIG_QCOM_GDSC=y +CONFIG_QCOM_GSBI=y +# CONFIG_QCOM_HFPLL is not set +# CONFIG_QCOM_MEMORY_DUMP_V2 is not set +# CONFIG_QCOM_MDT_LOADER is not set +CONFIG_QCOM_QFPROM=y +# CONFIG_QCOM_SPMI_TEMP_ALARM is not set +CONFIG_QCOM_RPM_CLK=y +# CONFIG_QCOM_RTB is not set +# CONFIG_QCOM_PM is not set +CONFIG_QCOM_SCM=y +CONFIG_QCOM_SCM_32=y +# CONFIG_HAVE_ARM_SMCCC is not set +CONFIG_QCA_SCM_RESTART_REASON=y +CONFIG_IPQ_TCSR=y +CONFIG_QCOM_QFPROM=y +# CONFIG_QCOM_SMD is not set +CONFIG_QCOM_SMEM=y +CONFIG_QCOM_SMEM_STATE=y +# CONFIG_QCOM_SMD is not set +CONFIG_QCOM_SMP2P=y +# CONFIG_QCOM_SPMI_VADC is not set +CONFIG_QCOM_TSENS=y +CONFIG_QCOM_TZ_LOG=y +CONFIG_QCOM_WDT=y +CONFIG_QMI_ENCDEC=y +CONFIG_RATIONAL=y +# CONFIG_RCU_BOOST is not set +CONFIG_RCU_CPU_STALL_TIMEOUT=21 +# CONFIG_RCU_EXPERT is not set +CONFIG_RCU_STALL_COMMON=y +CONFIG_RD_GZIP=y +CONFIG_REGMAP=y +# CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS is not set +CONFIG_REGMAP_MMIO=y +CONFIG_REGULATOR=y +# CONFIG_REGULATOR_CPR3 is not set +CONFIG_REGULATOR_GPIO=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_QCOM_RPM=y +CONFIG_REGULATOR_QCOM_SPMI=y +# CONFIG_REGULATOR_IPQ40XX is not set +# CONFIG_REGULATOR_RPM_SMD is not set +# CONFIG_REGULATOR_RPM_GLINK is not set +CONFIG_RELAY=y +CONFIG_REMOTEPROC=y +# CONFIG_IPQ807X_REMOTEPROC is not set +CONFIG_RESET_CONTROLLER=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_RTC_CLASS=y +CONFIG_NET_L3_MASTER_DEV=y +# CONFIG_RTC_DRV_CMOS is not set +# CONFIG_RTC_DRV_PM8XXX is not set +CONFIG_RWSEM_SPIN_ON_OWNER=y +CONFIG_RWSEM_XCHGADD_ALGORITHM=y +# CONFIG_SATA_AHCI is not set +CONFIG_SCHED_HRTICK=y +# CONFIG_SCSI is not set +# CONFIG_SCHED_INFO is not set +# CONFIG_SCSI_DMA is not set +# CONFIG_SERIAL_8250 is not set +# CONFIG_SERIAL_8250_CONSOLE is not set +# CONFIG_SERIAL_8250_DMA is not set +# CONFIG_SERIAL_AMBA_PL010 is not set +# CONFIG_SERIAL_AMBA_PL011 is not set +CONFIG_SERIAL_MSM=y +CONFIG_SERIAL_MSM_CONSOLE=y +# CONFIG_VIRTIO_CONSOLE is not set +CONFIG_SMP=y +CONFIG_SMP_ON_UP=y +# CONFIG_SND is not set +CONFIG_SND_DYNAMIC_MINORS=y +CONFIG_SND_MAX_CARDS=32 +CONFIG_SND_PROC_FS=y +# CONFIG_SND_COMPRESS_OFFLOAD is not set +CONFIG_SND_PCM=y +CONFIG_SND_SOC=y +# CONFIG_SND_SOC_APQ8016_SBC is not set +CONFIG_SND_SOC_I2C_AND_SPI=y +# CONFIG_SND_SOC_IPQ is not set +# CONFIG_SND_SOC_IPQ806X_LPAIF is not set +# CONFIG_SND_SOC_IPQ806X_PCM_RAW is not set +CONFIG_SND_SOC_IPQ_ADSS=y +CONFIG_SND_SOC_IPQ_CODEC=y +CONFIG_SND_SOC_IPQ_CPU_DAI=y +CONFIG_SND_SOC_IPQ_MBOX=y +CONFIG_SND_SOC_IPQ_PCM_I2S=y +CONFIG_SND_SOC_IPQ_PCM_RAW=y +CONFIG_SND_SOC_IPQ_PCM_SPDIF=y +CONFIG_SND_SOC_IPQ_PCM_TDM=y +CONFIG_SND_SOC_IPQ_STEREO=y +CONFIG_SND_SOC_QCOM=y +# CONFIG_SND_SOC_STORM is not set +CONFIG_SOUND=y +CONFIG_SPARSE_IRQ=y +CONFIG_SPI=y +CONFIG_SPI_MASTER=y +CONFIG_SPI_QUP=y +CONFIG_SPI_SPIDEV=y +# CONFIG_SPI_VSC7385 is not set +CONFIG_SPMI=y +CONFIG_SPMI_MSM_PMIC_ARB=y +CONFIG_SRCU=y +# CONFIG_STRIP_ASM_SYMS is not set +# CONFIG_STOPWATCH is not set +CONFIG_SUSPEND=y +CONFIG_SWCONFIG=y +CONFIG_SWIOTLB=y +# CONFIG_SWAP is not set +CONFIG_SWP_EMULATE=y +CONFIG_SYS_SUPPORTS_APM_EMULATION=y +CONFIG_THERMAL=y +CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y +CONFIG_THERMAL_GOV_STEP_WISE=y +CONFIG_THERMAL_GOV_USER_SPACE=y +CONFIG_THERMAL_HWMON=y +CONFIG_THERMAL_OF=y +CONFIG_THERMAL_WRITABLE_TRIPS=y +# CONFIG_THUMB2_KERNEL is not set +# CONFIG_TICK_CPU_ACCOUNTING is not set +CONFIG_IRQ_TIME_ACCOUNTING=y +CONFIG_TRACING_EVENTS_GPIO=y +CONFIG_UBIFS_FS=y +CONFIG_UBIFS_FS_ADVANCED_COMPR=y +CONFIG_UBIFS_FS_LZO=y +CONFIG_UBIFS_FS_XZ=y +CONFIG_UBIFS_FS_ZLIB=y +CONFIG_UEVENT_HELPER_PATH="" +CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" +CONFIG_UNINLINE_SPIN_UNLOCK=y +CONFIG_USB_GADGET=n +CONFIG_USB_SUPPORT=y +# CONFIG_USB_DWC3_OF_SIMPLE is not set +# CONFIG_USB_QCOM_8X16_PHY is not set +# CONFIG_USB_QCOM_KS_BRIDGE is not set +# CONFIG_USB_QCOM_QUSB_PHY is not set +# CONFIG_USB_QCOM_QMP_PHY is not set +# CONFIG_USB_QCA_M31_PHY is not set +# CONFIG_USB_EHCI_ROOT_HUB_TT is not set +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +# CONFIG_USB_OHCI_LITTLE_ENDIAN is not set +CONFIG_USE_OF=y +CONFIG_VDSO=y +CONFIG_VECTORS_BASE=0xffff0000 +CONFIG_VFP=y +CONFIG_VFPv3=y +CONFIG_VM_EVENT_COUNTERS=y +CONFIG_WATCHDOG_CORE=y +# CONFIG_WL_TI is not set +# CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set +CONFIG_XPS=y +CONFIG_XZ_DEC_ARM=y +CONFIG_XZ_DEC_BCJ=y +CONFIG_ZBOOT_ROM_BSS=0 +CONFIG_ZBOOT_ROM_TEXT=0 +CONFIG_ZLIB_DEFLATE=y +CONFIG_ZLIB_INFLATE=y +CONFIG_ZONE_DMA_FLAG=0 +CONFIG_QCOM_CACHE_DUMP=y +CONFIG_QCOM_CACHE_DUMP_ON_PANIC=y +# CONFIG_QCOM_RESTART_REASON is not set +# CONFIG_QCOM_DLOAD_MODE is not set +CONFIG_FW_AUTH=y +CONFIG_FW_AUTH_TEST=m +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_PUBLIC_KEY_ALGO_RSA=y +CONFIG_X509_CERTIFICATE_PARSER=y +CONFIG_PKCS7_MESSAGE_PARSER=n +CONFIG_KEYS=y +# CONFIG_SKB_RECYCLER is not set +CONFIG_SKB_RECYCLER_MULTI_CPU=y +# CONFIG_SKB_RECYCLER_PREALLOC is not set +# CONFIG_U_SERIAL_CONSOLE is not set +CONFIG_SCSI_SCAN_ASYNC=y +# CONFIG_NF_IPV6_DUMMY_HEADER is not set +# CONFIG_RMNET is not set +# CONFIG_RMNET_DATA is not set +# CONFIG_RMNET_CTL is not set +# CONFIG_MSM_SECURE_BUFFER is not set +# CONFIG_STAGING is not set +# CONFIG_ANDROID is not set +# CONFIG_ION is not set +# CONFIG_ION_DUMMY is not set +# CONFIG_ION_MSM is not set +# CONFIG_ION_TEST is not set +# CONFIG_CMA is not set +# CONFIG_CMA_DEBUG is not set +# CONFIG_CMA_DEBUGFS is not set +# CONFIG_DMA_CMA is not set +# CONFIG_CMA_AREAS is not set +# CONFIG_CMA_SIZE_MBYTES is not set +# CONFIG_CMA_SIZE_SEL_MBYTES is not set +# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set +# CONFIG_CMA_SIZE_SEL_MIN is not set +# CONFIG_CMA_SIZE_SEL_MAX is not set +# CONFIG_CMA_ALIGNMENT is not set +# CONFIG_ASHMEM is not set +# CONFIG_ANDROID_TIMED_OUTPUT is not set +# CONFIG_ANDROID_LOW_MEMORY_KILLER is not set +# CONFIG_SYNC is not set +# CONFIG_SW_SYNC is not set +# CONFIG_FSL_MC_BUS is not set +# CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS is not set +CONFIG_ALLOC_SKB_PAGE_FRAG_DISABLE=y +# CONFIG_MMAP_ALLOW_UNINITIALIZED is not set +# CONFIG_MAILBOX is not set +# CONFIG_MAILBOX_TEST is not set +# CONFIG_QCOM_APCS_IPC is not set +# CONFIG_QCOM_GLINK_SSR is not set +# CONFIG_QCOM_Q6V5_WCSS is not set +# CONFIG_QCOM_SYSMON is not set +# CONFIG_QRTR is not set +# CONFIG_QRTR_SMD is not set +# CONFIG_QRTR_TUN is not set +# CONFIG_RPMSG is not set +# CONFIG_RPMSG_QCOM_GLINK_RPM is not set +# CONFIG_RPMSG_VIRTIO is not set +# CONFIG_RPMSG_CHAR is not set +# CONFIG_RPMSG_QCOM_GLINK_SMEM is not set +# CONFIG_RPMSG_QCOM_SMD is not set +CONFIG_QCA_MINIDUMP=y +# CONFIG_QCA_MINIDUMP_DEBUG is not set +# CONFIG_QRTR_USB is not set +# CONFIG_QRTR_FIFO is not set +CONFIG_QRTR_MHI=y +CONFIG_MHI_BUS=y +# CONFIG_MHI_QTI is not set +# CONFIG_MHI_NETDEV is not set +# CONFIG_MHI_DEBUG is not set +# CONFIG_MHI_UCI is not set +# CONFIG_MHI_SATELLITE is not set +# CONFIG_DIAG_OVER_QRTR is not set +# CONFIG_MSM_ADSPRPC is not set +CONFIG_CRYPTO_MICHAEL_MIC=y +# CONFIG_ARCH_HAS_KCOV is not set +# CONFIG_KCOV is not set +# CONFIG_GCC_PLUGINS is not set +# CONFIG_QTI_Q6V5_ADSP is not set +# CONFIG_MSM_RPM_RPMSG is not set +# CONFIG_RPMSG_QCOM_GLINK_RPM is not set +# CONFIG_REGULATOR_RPM_GLINK is not set +# CONFIG_MTD_NAND_SERIAL is not set +# CONFIG_ARM_QTI_IPQ60XX_CPUFREQ is not set +# CONFIG_PAGE_SCOPE_MULTI_PAGE_READ is not set +# CONFIG_CRYPTO_NO_ZERO_LEN_HASH is not set +# CONFIG_CRYPTO_DISABLE_AES192_TEST is not set +# CONFIG_QTI_EUD is not set +# CONFIG_EUD_EXTCON_SUPPORT is not set +# CONFIG_CLK_TEST_5018 is not set +CONFIG_MAP_E_SUPPORT=y +# CONFIG_IPQ_FLASH_16M_PROFILE is not set +# CONFIG_QGIC2_MSI is not set +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_LEDS_GPIO=y +# CONFIG_ARCH_IPQ256M is not set +CONFIG_SKB_FIXED_SIZE_2K=y +# CONFIG_IPQ_MEM_PROFILE is not set +# CONFIG_VIRTIO_NET is not set +# CONFIG_QCA_85XX_SWITCH is not set +CONFIG_AQ_PHY=y +CONFIG_DIAG_CHAR=y +# CONFIG_HW_RANDOM_VIRTIO is not set +# CONFIG_BOOTCONFIG_PARTITION is not set +# CONFIG_CRYPTO_DEV_QCEDEV is not set +# CONFIG_CRYPTO_DEV_QCRYPTO is not set +# CONFIG_MHI_BUS_TEST is not set diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts new file mode 100644 index 000000000..5f2d4fb03 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts new file mode 100644 index 000000000..ffd789922 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts new file mode 100644 index 000000000..531510a76 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts new file mode 100644 index 000000000..5a44b9a3e --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts new file mode 100644 index 000000000..8edd1d817 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts new file mode 100644 index 000000000..470e437e4 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts new file mode 100644 index 000000000..1250eb19f --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts" +#include "qcom-ipq6018.dtsi" diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts new file mode 100644 index 000000000..d1d3f3755 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts new file mode 100644 index 000000000..1527f81a4 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts new file mode 100644 index 000000000..f3b25e263 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts new file mode 100644 index 000000000..7cfd66ba0 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts new file mode 100644 index 000000000..4274b6c70 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts new file mode 100644 index 000000000..372f29df3 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf196.dts b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf196.dts new file mode 100644 index 000000000..f8d31a052 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf196.dts @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-wf196.dts" + +/ { + soc { + pmu { + compatible = "arm,cortex-a7-pmu"; + interrupts = ; + }; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts new file mode 100644 index 000000000..4c4d6f1b8 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts @@ -0,0 +1,423 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Cigtech WF-188"; + compatible = "cig,wf188", "qcom,ipq6018-cp03", "qcom,ipq6018"; + interrupt-parent = <&intc>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; + bootargs-append = " swiotlb=1 coherent_pool=2M"; + }; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled only in default memory profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio22"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_5g { + pins = "gpio25"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio24"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_eth { + pins = "gpio18"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_pwr { + pins = "gpio16"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + uart2_pins: uart2_pins { + mux { + pins = "gpio57", "gpio58"; + function = "blsp4_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&soc { + mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0>; + status = "ok"; + phy0: ethernet-phy@0 { + reg = <3>; + }; + phy1: ethernet-phy@1 { + reg = <4>; + }; + }; + + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x08>; /* lan port bitmap */ + switch_wan_bmp = <0x10>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@3 { + port_id = <3>; + phy_address = <4>; + }; + port@4 { + port_id = <4>; + phy_address = <3>; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + wps { + label = "reset"; + linux,code = ; + gpios = <&tlmm 22 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "green:wifi5"; + gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:5g"; + default-state = "off"; + }; + led@24 { + label = "green:wifi2"; + gpios = <&tlmm 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:2g"; + default-state = "off"; + }; + led@18 { + label = "green:wan"; + gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:eth"; + default-state = "off"; + }; + led_power: led@16 { + label = "green:power"; + gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:power"; + default-state = "off"; + }; + }; + + gpio-watchdog { + compatible = "linux,wdt-gpio"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + hw_algo = "toggle"; + hw_margin_ms = <5000>; + always-running; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&uart2_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; + +&cpu0_opp_table { + compatible = "operating-points-v2"; + opp-shared; + opp03 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <3>; + clock-latency-ns = <200000>; + }; + /delete-node/ opp04; + /delete-node/ opp05; + /delete-node/ opp06; +}; + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) +&qseecom { + mem-start = <0x49B00000>; + mem-size = <0x600000>; + status = "ok"; +}; +#endif diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts new file mode 100644 index 000000000..4956502ce --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts @@ -0,0 +1,423 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Cigtech WF-188n"; + compatible = "cig,wf188n", "qcom,ipq6018-cp03", "qcom,ipq6018"; + interrupt-parent = <&intc>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; + bootargs-append = " swiotlb=1 coherent_pool=2M"; + }; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled only in default memory profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio22"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_5g { + pins = "gpio25"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio24"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_eth { + pins = "gpio18"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_pwr { + pins = "gpio16"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + uart2_pins: uart2_pins { + mux { + pins = "gpio57", "gpio58"; + function = "blsp4_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&soc { + mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0>; + status = "ok"; + phy0: ethernet-phy@0 { + reg = <3>; + }; + phy1: ethernet-phy@1 { + reg = <4>; + }; + }; + + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x08>; /* lan port bitmap */ + switch_wan_bmp = <0x10>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@3 { + port_id = <3>; + phy_address = <4>; + }; + port@4 { + port_id = <4>; + phy_address = <3>; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + wps { + label = "reset"; + linux,code = ; + gpios = <&tlmm 22 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "green:wifi5"; + gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:5g"; + default-state = "off"; + }; + led@24 { + label = "green:wifi2"; + gpios = <&tlmm 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:2g"; + default-state = "off"; + }; + led@18 { + label = "green:wan"; + gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:eth"; + default-state = "off"; + }; + led_power: led@16 { + label = "green:power"; + gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:power"; + default-state = "on"; + }; + }; + + gpio-watchdog { + compatible = "linux,wdt-gpio"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + hw_algo = "toggle"; + hw_margin_ms = <5000>; + always-running; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&uart2_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; + +&cpu0_opp_table { + compatible = "operating-points-v2"; + opp-shared; + opp03 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <3>; + clock-latency-ns = <200000>; + }; + /delete-node/ opp04; + /delete-node/ opp05; + /delete-node/ opp06; +}; + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) +&qseecom { + mem-start = <0x49B00000>; + mem-size = <0x600000>; + status = "ok"; +}; +#endif diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-edgecore.dtsi b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-edgecore.dtsi new file mode 100644 index 000000000..04e65d608 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-edgecore.dtsi @@ -0,0 +1,542 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + compatible = "qcom,ipq6018-cp01", "qcom,ipq6018"; + interrupt-parent = <&intc>; + qcom,msm-id = <0x192 0x0>, <0x193 0x0>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + sdhc2 = &sdhc_2; + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + ethernet2 = "/soc/dp3"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; +#ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; +#else + bootargs-append = " swiotlb=1 coherent_pool=2M"; +#endif + }; + +}; + +&tlmm { + pinctrl-0 = <&sd_ldo_pins>; + pinctrl-names = "default"; + + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + sd_ldo_pins: sd_ldo_pins { + mux { + pins = "gpio66"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-low; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + spi_1_pins: spi_1_pins { + mux { + pins = "gpio69", "gpio71", "gpio72"; + function = "blsp1_spi"; + drive-strength = <8>; + bias-pull-down; + }; + spi_cs { + pins = "gpio70"; + function = "blsp1_spi"; + drive-strength = <8>; + bias-disable; + }; + quartz_interrupt { + pins = "gpio78"; + function = "gpio"; + input; + bias-disable; + }; + quartz_reset { + pins = "gpio79"; + function = "gpio"; + output-low; + bias-disable; + }; + + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + sd_pins: sd_pins { + mux { + pins = "gpio62"; + function = "sd_card"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + extcon_usb_pins: extcon_usb_pins { + mux { + pins = "gpio26"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio9"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + reset_button { + pins = "gpio19"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + mux_3 { + pins = "gpio77"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_pwr { + pins = "gpio74"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_5g { + pins = "gpio35"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_wan { + pins = "gpio62"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_lan1 { + pins = "gpio61"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_lan2 { + pins = "gpio63"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_bt { + pins = "gpio73"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio71", "gpio72", "gpio69", "gpio70"; + function = "blsp1_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio51"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio53"; + function = "pta1_0"; + drive-strength = <6>; + bias-pull-down; + }; + mux_2 { + pins = "gpio52"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + }; +}; + +&soc { + extcon_usb: extcon_usb { + pinctrl-0 = <&extcon_usb_pins>; + pinctrl-names = "default"; + id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; + status = "ok"; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; + status = "ok"; + phy0: ethernet-phy@0 { + reg = <0x1c>; + }; + phy1: ethernet-phy@1 { + reg = <3>; + }; + phy2: ethernet-phy@2 { + reg = <4>; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x1e>; /* lan port bitmap */ + switch_wan_bmp = <0x20>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@2 { + port_id = <3>; + phy_address = <4>; + }; + port@1 { + port_id = <4>; + phy_address = <3>; + }; + port@0 { + port_id = <5>; + phy_address = <0x1c>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + /*wps { + label = "wps"; + linux,code = ; + gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + };*/ + reset { + label = "reset"; + linux,code = ; + gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led_power: led@74 { + label = "green:power"; + gpios = <&tlmm 74 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led_pwr"; + default-state = "on"; + }; + led@35 { + label = "green:wifi5"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led_5g"; + default-state = "off"; + }; + led@37 { + label = "green:wifi2"; + gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led_2g"; + default-state = "off"; + }; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&hsuart_pins &btcoex_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; + +&spi_1 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_1_pins>; + pinctrl-names = "default"; + cs-select = <0>; + quartz-reset-gpio = <&tlmm 79 1>; + status = "disabled"; + spidev1: spi@1 { + compatible = "qca,spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb2 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; + +&pcie_phy { + status = "ok"; +}; + +&pcie0 { + status = "ok"; +}; + +&qpic_lcd { + status = "ok"; +}; + +&qpic_lcd_panel { + status = "ok"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi new file mode 100644 index 000000000..7c67cf7ad --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi @@ -0,0 +1,478 @@ +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + compatible = "qcom,ipq6018-cp01", "qcom,ipq6018"; + interrupt-parent = <&intc>; + qcom,msm-id = <0x192 0x0>, <0x193 0x0>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + sdhc2 = &sdhc_2; + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; +#ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; +#else + bootargs-append = " swiotlb=1 coherent_pool=2M"; +#endif + }; + +}; + +&tlmm { + pinctrl-0 = <&sd_ldo_pins &hfcl_gpio>; + pinctrl-names = "default"; + + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + sd_ldo_pins: sd_ldo_pins { + mux { + pins = "gpio66"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + output-low; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + sd_pins: sd_pins { + mux { + pins = "gpio62"; + function = "sd_card"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + extcon_usb_pins: extcon_usb_pins { + mux { + pins = "gpio26"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + reset_button { + pins = "gpio53"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + mux_3 { + pins = "gpio77"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_5g { + pins = "gpio60"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio61"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + hfcl_gpio: hfcl_gpio { + mux_0 { + pins = "gpio25"; + function = "gpio"; + drive-strength = <8>; + output-low; + }; + mux_1 { + pins = "gpio29"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_2 { + pins = "gpio33"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_3 { + pins = "gpio34"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_4 { + pins = "gpio35"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_5 { + pins = "gpio59"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_6 { + pins = "gpio67"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_7 { + pins = "gpio70"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + mux_8 { + pins = "gpio79"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + uniphy_pins: uniphy_pinmux { + mux { + pins = "gpio63"; + function = "rx0"; + bias-disable; + }; + sfp_tx { + pins = "gpio48"; + function = "gpio"; + driver-strength = <8>; + bias-pull-down; + output-low; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio71", "gpio72"; + function = "blsp1_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + +}; + +&soc { + extcon_usb: extcon_usb { + pinctrl-0 = <&extcon_usb_pins>; + pinctrl-names = "default"; + id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; + status = "disabled"; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 77 0>; + status = "ok"; + phy0: ethernet-phy@0 { + reg = <4>; + }; + phy1: ethernet-phy@1 { + reg = <30>; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a003000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <30>; + phy-mode = "sgmii"; + }; + + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <30>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + + ess-switch@3a000000 { + pinctrl-0 = <&uniphy_pins>; + pinctrl-names = "default"; + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x10>; /* lan port bitmap */ + switch_wan_bmp = <0x20>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0xf>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0x14>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@0 { + port_id = <4>; + phy_address = <4>; + }; + port@2 { + port_id = <5>; + phy_address = <30>; + phy_i2c_address = <30>; + phy-i2c-mode; + port_mac_sel = "GMAC_PORT"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + reset { + label = "reset"; + linux,code = ; + gpios = <&tlmm 53 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + i2c_2: i2c@78b9000 { + compatible = "qcom,i2c-qup-v2.2.1"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x78b9000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_AHB_CLK>, + <&gcc GCC_BLSP1_QUP5_I2C_APPS_CLK>; + clock-names = "iface", "core"; + clock-frequency = <400000>; + dmas = <&blsp_dma 21>, <&blsp_dma 20>; + dma-names = "rx", "tx"; + status = "disabled"; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb2 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; + +&pcie_phy { + status = "disabled"; +}; + +&pcie0 { + status = "disabled"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts new file mode 100644 index 000000000..7506115f9 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts @@ -0,0 +1,441 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "EdgeCore EAP101"; + compatible = "edgecore,eap101", "qcom,ipq6018-cp01", "qcom,ipq6018"; + interrupt-parent = <&intc>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; + bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; + }; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled only in default memory profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + extcon_usb_pins: extcon_usb_pins { + mux { + pins = "gpio26"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio19"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_pwr { + pins = "gpio74"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_5g { + pins = "gpio35"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + uart2_pins: uart2_pins { + mux { + pins = "gpio57", "gpio58"; + function = "blsp4_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&soc { + extcon_usb: extcon_usb { + pinctrl-0 = <&extcon_usb_pins>; + pinctrl-names = "default"; + id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; + status = "ok"; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; + status = "ok"; + phy0: ethernet-phy@0 { + reg = <0x1c>; + }; + phy1: ethernet-phy@1 { + reg = <3>; + }; + phy2: ethernet-phy@2 { + reg = <4>; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x1e>; /* lan port bitmap */ + switch_wan_bmp = <0x20>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@2 { + port_id = <3>; + phy_address = <4>; + }; + port@1 { + port_id = <4>; + phy_address = <3>; + }; + port@0 { + port_id = <5>; + phy_address = <0x1c>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + wps { + label = "reset"; + linux,code = ; + gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "green:wifi5"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:5g"; + default-state = "off"; + }; + led@24 { + label = "green:wifi2"; + gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wf188:green:2g"; + default-state = "off"; + }; + led_power: led@16 { + label = "green:led_pwr"; + gpios = <&tlmm 74 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + led@61 { + label = "green:lan1"; + gpios = <&tlmm 61 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + led@62 { + label = "green:wan"; + gpios = <&tlmm 62 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + led@63 { + label = "green:lan2"; + gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&uart2_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb2 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts new file mode 100644 index 000000000..a64ec25a8 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts @@ -0,0 +1,155 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018-cp01-hfcl.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" + +/ { + model = "HFCL ION4Xe"; + compatible = "hfcl,ion4xe", "qcom,ipq6018-cp01", "qcom,ipq6018"; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + i2c_1_pins: i2c_1_pins { + mux { + pins = "gpio42", "gpio43"; + function = "blsp2_i2c"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + i2c_2_pins: i2c_2_pins { + mux { + pins = "gpio55", "gpio56"; + function = "blsp4_i2c"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&i2c_1 { + pinctrl-0 = <&i2c_1_pins>; + pinctrl-names = "default"; + status = "ok"; + + lm75@48 { + compatible = "lm75"; + reg = <0x48>; + status = "okay"; + }; +}; + +&i2c_2 { + pinctrl-0 = <&i2c_2_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&sdhc_2 { + pinctrl-0 = <&sd_pins>; + pinctrl-names = "default"; + cd-gpios = <&tlmm 62 1>; + sd-ldo-gpios = <&tlmm 66 0>; + vqmmc-supply = <&ipq6018_l2_corner>; + status = "ok"; +}; + +&soc { + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@60 { + label = "blue:wifi5"; + gpios = <&tlmm 60 GPIO_ACTIVE_LOW>; + linux,default-trigger = "led_5g"; + default-state = "off"; + }; + led@61 { + label = "blue:wifi2"; + gpios = <&tlmm 61 GPIO_ACTIVE_LOW>; + linux,default-trigger = "led_2g"; + default-state = "off"; + }; + }; +}; + +&tlmm { + leds_pins: leds_pins { + led_5g { + pins = "gpio60"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio61"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) +&qseecom { + mem-start = <0x49B00000>; + mem-size = <0x600000>; + status = "ok"; +}; +#endif diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts new file mode 100644 index 000000000..88b609f37 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts @@ -0,0 +1,156 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018-cp01-hfcl.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" + +/ { + model = "HFCL ION4Xi"; + compatible = "hfcl,ion4xi", "qcom,ipq6018-cp01", "qcom,ipq6018"; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + i2c_1_pins: i2c_1_pins { + mux { + pins = "gpio42", "gpio43"; + function = "blsp2_i2c"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + i2c_2_pins: i2c_2_pins { + mux { + pins = "gpio55", "gpio56"; + function = "blsp4_i2c"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&i2c_1 { + pinctrl-0 = <&i2c_1_pins>; + pinctrl-names = "default"; + status = "ok"; + + lm75@48 { + compatible = "lm75"; + reg = <0x48>; + status = "okay"; + }; +}; + +&i2c_2 { + pinctrl-0 = <&i2c_2_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&sdhc_2 { + pinctrl-0 = <&sd_pins>; + pinctrl-names = "default"; + cd-gpios = <&tlmm 62 1>; + sd-ldo-gpios = <&tlmm 66 0>; + vqmmc-supply = <&ipq6018_l2_corner>; + status = "ok"; +}; + +&soc { + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@60 { + label = "blue:wifi5"; + gpios = <&tlmm 32 GPIO_ACTIVE_LOW>; + linux,default-trigger = "led_5g"; + default-state = "off"; + }; + led@61 { + label = "blue:wifi2"; + gpios = <&tlmm 31 GPIO_ACTIVE_LOW>; + linux,default-trigger = "led_2g"; + default-state = "off"; + }; + }; +}; + +&tlmm { + leds_pins: leds_pins { + led_5g { + pins = "gpio32"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio31"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + + +/* TZAPP is enabled in default memory profile only */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) +&qseecom { + mem-start = <0x49B00000>; + mem-size = <0x600000>; + status = "ok"; +}; +#endif diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-ion.dtsi b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-ion.dtsi new file mode 100644 index 000000000..0f637134e --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-ion.dtsi @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + adsp_mem: adsp_region { + compatible = "shared-dma-pool"; + alloc-ranges = <0 0x00000000 0 0xffffffff>; + reusable; + alignment = <0 0x400000>; + size = <0 0x800000>; + }; + }; +}; + +&soc { + ion: qcom,ion { + compatible = "qcom,msm-ion"; + #address-cells = <1>; + #size-cells = <0>; + + system_heap: qcom,ion-heap@25 { + reg = <25>; + qcom,ion-heap-type = "SYSTEM"; + }; + + qcom,ion-heap@22 { /* ADSP HEAP */ + reg = <22>; + memory-region = <&adsp_mem>; + qcom,ion-heap-type = "DMA"; + }; + }; +}; +#endif diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts new file mode 100644 index 000000000..a00c106ad --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts @@ -0,0 +1,492 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Wallys DR6018 V4"; + compatible = "wallys,dr6018-v4", "qcom,ipq6018-cp01", "qcom,ipq6018"; + interrupt-parent = <&intc>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + ethernet2 = "/soc/dp3"; + ethernet3 = "/soc/dp4"; + ethernet4 = "/soc/dp5"; + + sdhc2 = "/soc/sdhci_sd@7804000"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; + bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; + }; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled only in default memory profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + sd_pins: sd_pins { + mux { + pins = "gpio62"; + function = "sd_card"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + extcon_usb_pins: extcon_usb_pins { + mux { + pins = "gpio26"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio19"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_pwr { + pins = "gpio74"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_5g { + pins = "gpio35"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + uart2_pins: uart2_pins { + mux { + pins = "gpio57", "gpio58"; + function = "blsp4_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&soc { + extcon_usb: extcon_usb { + pinctrl-0 = <&extcon_usb_pins>; + pinctrl-names = "default"; + id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; + status = "ok"; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; + status = "ok"; + ethernet-phy@3 { + reg = <0x03>; + }; + + ethernet-phy@4 { + reg = <0x18>; + }; + + ethernet-phy@1 { + reg = <0x01>; + }; + + ethernet-phy@2 { + reg = <0x02>; + }; + + ethernet-phy@0 { + reg = <0x00>; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <1>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <2>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <1>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <2>; + phy-mode = "sgmii"; + }; + + dp4 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + + dp5 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001800 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0x18>; + phy-mode = "sgmii"; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x3c>; /* lan port bitmap */ + switch_wan_bmp = <0x2>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@1 { + phy_address = <0x01>; + port_id = <0x02>; + }; + + port@0 { + phy_address = <0x00>; + port_id = <0x01>; + }; + + port@2 { + phy_address = <0x02>; + port_id = <0x03>; + }; + + port@3 { + phy_address = <0x03>; + port_id = <0x04>; + }; + + port@4 { + phy_address = <0x18>; + port_id = <0x05>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + reset { + label = "reset"; + linux,code = ; + gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "green:wifi5"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:5g"; + default-state = "off"; + }; + led@24 { + label = "green:wifi2"; + gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:2g"; + default-state = "off"; + }; + led_power: led@16 { + label = "green:led_pwr"; + gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&uart2_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb2 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; + +&sdhc_2 { + pinctrl-0 = <&sd_pins>; + pinctrl-names = "default"; + cd-gpios = <&tlmm 62 1>; + sd-ldo-gpios = <&tlmm 66 0>; + //vqmmc-supply = <&ipq6018_l2_corner>; + status = "ok"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts new file mode 100644 index 000000000..12065b62f --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts @@ -0,0 +1,441 @@ +/dts-v1/; +/* + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "qcom-ipq6018.dtsi" +#include "qcom-ipq6018-rpm-regulator.dtsi" +#include "qcom-ipq6018-cpr-regulator.dtsi" +#include "qcom-ipq6018-cp-cpu.dtsi" +#include +#include + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Wallys DR6018"; + compatible = "wallys,dr6018", "qcom,ipq6018-cp01", "qcom,ipq6018"; + interrupt-parent = <&intc>; + + aliases { + serial0 = &blsp1_uart3; + serial1 = &blsp1_uart2; + + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; + bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; + }; + + /* + * +=========+==============+========================+ + * | | | | + * | Region | Start Offset | Size | + * | | | | + * +--------+--------------+-------------------------+ + * | | | | + * | | | | + * | | | | + * | | | | + * | Linux | 0x41000000 | 139MB | + * | | | | + * | | | | + * | | | | + * +--------+--------------+-------------------------+ + * | TZ App | 0x49B00000 | 6MB | + * +--------+--------------+-------------------------+ + * + * From the available 145 MB for Linux in the first 256 MB, + * we are reserving 6 MB for TZAPP. + * + * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi + * for memory layout. + */ + +/* TZAPP is enabled only in default memory profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) + reserved-memory { + tzapp:tzapp@49B00000 { /* TZAPPS */ + no-map; + reg = <0x0 0x49B00000 0x0 0x00600000>; + }; + }; +#endif +}; + +&tlmm { + uart_pins: uart_pins { + mux { + pins = "gpio44", "gpio45"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-pull-down; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-pull-down; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-pull-down; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-pull-down; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-pull-down; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-pull-down; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-pull-down; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-pull-down; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + extcon_usb_pins: extcon_usb_pins { + mux { + pins = "gpio26"; + function = "gpio"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio19"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio64"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio65"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio75"; + function = "gpio"; + bias-pull-up; + }; + }; + + leds_pins: leds_pins { + led_pwr { + pins = "gpio74"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_5g { + pins = "gpio35"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led_2g { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + uart2_pins: uart2_pins { + mux { + pins = "gpio57", "gpio58"; + function = "blsp4_uart"; + drive-strength = <8>; + bias-pull-down; + }; + }; +}; + +&soc { + extcon_usb: extcon_usb { + pinctrl-0 = <&extcon_usb_pins>; + pinctrl-names = "default"; + id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; + status = "ok"; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; + status = "ok"; + ethernet-phy@3 { + reg = <0x03>; + }; + + ethernet-phy@4 { + reg = <0x18>; + }; + + ethernet-phy@1 { + reg = <0x01>; + }; + + ethernet-phy@2 { + reg = <0x02>; + }; + + ethernet-phy@0 { + reg = <0x00>; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <1>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <2>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <1>; + phy-mode = "sgmii"; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x4>; /* lan port bitmap */ + switch_wan_bmp = <0x2>; /* wan port bitmap */ + switch_inner_bmp = <0xc0>; /*inner port bitmap*/ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ + qcom,port_phyinfo { + port@1 { + phy_address = <0x01>; + port_id = <0x02>; + }; + + port@0 { + phy_address = <0x00>; + port_id = <0x01>; + }; + + port@2 { + phy_address = <0x02>; + port_id = <0x03>; + }; + + port@3 { + phy_address = <0x03>; + port_id = <0x04>; + }; + + port@4 { + phy_address = <0x18>; + port_id = <0x05>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + reset { + label = "reset"; + linux,code = ; + gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + + /* wps { + label = "wps"; + linux,code = <>; + gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + };*/ + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "green:wifi5"; + gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:5g"; + default-state = "off"; + }; + led@24 { + label = "green:wifi2"; + gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:2g"; + default-state = "off"; + }; + led_power: led@16 { + label = "green:led_pwr"; + gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "green:power"; + default-state = "off"; + }; + }; +}; + +&blsp1_uart3 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&blsp1_uart2 { + pinctrl-0 = <&uart2_pins>; + pinctrl-names = "default"; + dmas = <&blsp_dma 2>, + <&blsp_dma 3>; + dma-names = "tx", "rx"; + status = "ok"; +}; +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb2 { + status = "ok"; +}; + +&usb3 { + status = "ok"; +}; + +&nss_crypto { + status = "ok"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-audio.dtsi b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-audio.dtsi new file mode 100644 index 000000000..7601e169b --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-audio.dtsi @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include +#include + + +&soc { + adss_clk: qcom,adss_clk@7700000 { + compatible = "qcom,adss-ipq807x"; + reg = <0x7700100 0x200>; + #clock-cells = <0x1>; + }; + + audio: audio@7700000 { + compatible = "qca,ipq8074-audio-adss"; + reg = <0x7700000 0x34>; + resets = <&gcc GCC_ADSS_BCR>; + reset-names = "blk_rst"; + }; +}; + +&clocks { + audio_rxbpad_clk { + compatible = "fixed-clock"; + clock-frequency = <254>; + #clock-cells = <0>; + }; + + audio_txbpad_clk { + compatible = "fixed-clock"; + clock-frequency = <254>; + #clock-cells = <0>; + }; + + audio_rxmpad_clk { + compatible = "fixed-clock"; + clock-frequency = <252>; + #clock-cells = <0>; + }; + + audio_txmpad_clk { + compatible = "fixed-clock"; + clock-frequency = <252>; + #clock-cells = <0>; + }; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts new file mode 100644 index 000000000..d04cb1020 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts @@ -0,0 +1,918 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-ac-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Edgecore EAP102"; + compatible = "edgecore,eap102", "qcom,ipq807x-ac02", "qcom,ipq807x"; + qcom,msm-id = <0x178 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet1 = "/soc/dp5"; + ethernet0 = "/soc/dp6"; + + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ + init=/init"; + #ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; + #else + bootargs-append = " swiotlb=1 coherent_pool=2M"; + #endif + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio64"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio65"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio33"; + function = "gpio"; + bias-pull-up; + }; + mux_3 { + pins = "gpio44"; + function = "gpio"; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + button_pins: button_pins { + + reset_button { + pins = "gpio66"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + led_pins: led_pins { + led_pwr { + pins = "gpio46"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + led_2g { + pins = "gpio47"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + led_5g { + pins = "gpio48"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + led_bt { + pins = "gpio50"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + usb_mux_sel_pins: usb_mux_pins { + mux { + pins = "gpio27"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + pcie0_pins: pcie_pins { + pcie0_rst { + pins = "gpio58"; + function = "pcie0_rst"; + drive-strength = <8>; + bias-pull-down; + }; + pcie0_wake { + pins = "gpio59"; + function = "pcie0_wake"; + drive-strength = <8>; + bias-pull-down; + }; + }; + +}; + +&soc { + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset_button"; + linux,code = ; + gpios = <&tlmm 66 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1 &tlmm 44 1>; + compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; + phy0: ethernet-phy@0 { + reg = <0>; + }; + phy1: ethernet-phy@1 { + reg = <1>; + }; + phy2: ethernet-phy@2 { + reg = <2>; + }; + phy3: ethernet-phy@3 { + reg = <3>; + }; + phy4: ethernet-phy@4 { + reg = <24>; + }; + phy5: ethernet-phy@5 { + reg = <28>; + }; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x3e>; /* lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + qcom,port_phyinfo { + port@0 { + port_id = <1>; + phy_address = <0>; + }; + port@1 { + port_id = <2>; + phy_address = <1>; + }; + port@2 { + port_id = <3>; + phy_address = <2>; + }; + port@3 { + port_id = <4>; + phy_address = <3>; + }; + port@4 { + port_id = <5>; + phy_address = <24>; + port_mac_sel = "QGMAC_PORT"; + }; + port@5 { + port_id = <6>; + phy_address = <28>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; +/* + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <1>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <2>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <1>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <2>; + phy-mode = "sgmii"; + }; + + dp4 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; +*/ + dp6 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a001800 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; + + dp5 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001a00 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <24>; + phy-mode = "sgmii"; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + led_pwr { + label = "green:wan"; + gpios = <&tlmm 46 GPIO_ACTIVE_HIGH>; + default-state = "off"; + linux,default-trigger = "led_pwr"; + }; + + led_2g { + label = "green:wifi2"; + gpio = <&tlmm 47 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led_5g { + label = "green:wifi5"; + gpio = <&tlmm 48 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led_power: led_bt { + gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; + label = "green:power"; + default-state = "on"; + linux,default-trigger = "led_bt"; + }; + }; + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x18>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&serial_blsp2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&nss0 { + qcom,low-frequency = <187200000>; + qcom,mid-frequency = <748800000>; + qcom,max-frequency = <1497600000>; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + status = "disabled"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; + +&ledc { + status = "disabled"; +}; + +&pcie0 { + status = "ok"; +}; + +&pcie1 { + status = "disabled"; +}; + +&glink_rpm { + status = "disabled"; +}; + +&apc_cpr { + /* Same CPR configuration as OAK */ + compatible = "qcom,cpr4-ipq817x-apss-regulator"; + + thread@0 { + apc_vreg: regulator { + regulator-min-microvolt = <1>; + regulator-max-microvolt = <2>; + qcom,cpr-fuse-corners = <2>; + qcom,cpr-corners = <3>; + qcom,cpr-speed-bin-corners = <3>; + qcom,cpr-corner-fmax-map = <1 3>; + + qcom,cpr-voltage-ceiling = + <840000 904000 944000>; + qcom,cpr-voltage-floor = + <592000 648000 712000>; + qcom,corner-frequencies = + <1017600000 1382400000 1382400000>; + + qcom,cpr-open-loop-voltage-fuse-adjustment-0 = + /* Speed bin 0; CPR rev 0..7 */ + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>; + + qcom,cpr-open-loop-voltage-fuse-adjustment-1 = + /* Speed bin 0; CPR rev 0..7 */ + < 0 0>, + < 0 0>, + < 0 0>, + < 20000 26000>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>; + + qcom,cpr-open-loop-voltage-fuse-adjustment-v2-0 = + /* Speed bin 0; CPR rev 0..7 */ + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>; + + qcom,cpr-open-loop-voltage-fuse-adjustment-v2-1 = + /* Speed bin 0; CPR rev 0..7 */ + < 0 0>, + < 0 7000>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>, + < 0 0>; + + qcom,cpr-floor-to-ceiling-max-range = + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>, + < 40000 40000 40000>; + }; + }; +}; + +&npu_cpr { + status = "disabled"; +}; + +&nss0 { + npu-supply = <&dummy_reg>; + mx-supply = <&dummy_reg>; +}; + +&wifi0 { + qcom,board_id = <0x92>; +}; + +&wifi1 { + qcom,board_id = <0x292>; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts new file mode 100644 index 000000000..d733513c3 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts @@ -0,0 +1,716 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "Edgecore EAP106"; + compatible = "edgecore,eap106", "qcom,ipq807x-hk02", "qcom,ipq807x"; + qcom,msm-id = <0x143 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + }; + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw init=/init"; + #ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; + #else + bootargs-append = " swiotlb=1 coherent_pool=2M"; + #endif + }; +}; + +&tlmm { + leds_pins: leds_pinmux { + + led1_yellow { + pins = "gpio25"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led1_green { + pins = "gpio28"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_amber { + pins = "gpio29"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_blue { + pins = "gpio32"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + i2c_0_pins: i2c_0_pinmux { + mux { + pins = "gpio42", "gpio43"; + function = "blsp1_i2c"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + data_8 { + pins = "gpio16"; + function = "qpic_pad8"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", + "gpio9", "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio46", "gpio47", "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + output-low; + }; + mux_1 { + pins = "gpio51"; + function = "gpio"; + drive-strength = <8>; + bias-disable; + output-high; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio57"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + uniphy_pins: uniphy_pinmux { + mux { + pins = "gpio60"; + function = "rx2"; + bias-disable; + }; + }; + cnss_wlan_en_active: cnss_wlan_en_active { + mux { + pins = "gpio57"; + function = "gpio"; + drive-strength = <16>; + output-high; + bias-pull-up; + }; + }; + + cnss_wlan_en_sleep: cnss_wlan_en_sleep { + mux { + pins = "gpio57"; + function = "gpio"; + drive-strength = <2>; + output-low; + bias-pull-down; + }; + }; +}; + +&soc { + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "wps"; + linux,code = ; + gpios = <&tlmm 57 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + + led@25 { + label = "led1_yellow"; + gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led1_yellow"; + default-state = "off"; + }; + + led@28 { + label = "led1_green"; + gpios = <&tlmm 28 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led1_green"; + default-state = "off"; + }; + + led@29 { + label = "led2_amber"; + gpios = <&tlmm 29 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led2_amber"; + default-state = "off"; + }; + + led@32 { + label = "led2_blue"; + gpio = <&tlmm 32 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led2_blue"; + default-state = "off"; + }; + }; + + mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0>; + phy0: ethernet-phy@0 { + reg = <0>; + }; + phy1: ethernet-phy@1 { + reg = <1>; + }; + phy2: ethernet-phy@2 { + reg = <2>; + }; + phy3: ethernet-phy@3 { + reg = <3>; + }; + phy4: ethernet-phy@4 { + reg = <4>; + }; + phy5: ethernet-phy@5 { + compatible ="ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; + }; + + ess-switch@3a000000 { + pinctrl-0 = <&uniphy_pins>; + pinctrl-names = "default"; + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x30>; /* lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + group@1 { + sp = <61>; + cfg = <1 33 1 33>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + ucast_loop_pri = <16>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a007000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <8>; + phy-mode = "sgmii"; + }; + wifi3: wifi3@f00000 { + compatible = "qcom,cnss-qcn9000"; + wlan-en-gpio = <&tlmm 57 0>; + pinctrl-names = "wlan_en_active", "wlan_en_sleep"; + pinctrl-0 = <&cnss_wlan_en_active>; + pinctrl-1 = <&cnss_wlan_en_sleep>; + status = "disabled"; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&serial_blsp2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + pinctrl-0 = <&i2c_0_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&pcie0 { + status = "disabled"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts new file mode 100644 index 000000000..0f34156f2 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts @@ -0,0 +1,754 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "TP-Link EX227"; + compatible = "tplink,ex227", "qcom,ipq807x"; + qcom,msm-id = <0x143 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ + init=/init"; + bootargs-append = " swiotlb=1"; + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio64"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio65"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + mux_2 { + pins = "gpio66"; + function = "pta1_0"; + drive-strength = <6>; + bias-pull-down; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio25"; + function = "gpio"; + bias-pull-up; + }; + mux_3 { + pins = "gpio37"; + function = "gpio"; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_3_pins: spi_3_pins { + mux { + pins = "gpio52", "gpio53"; + function = "blsp3_spi"; + drive-strength = <8>; + bias-disable; + }; + spi_cs { + pins = "gpio22"; + function = "blsp3_spi2"; + drive-strength = <8>; + bias-disable; + }; + quartz_interrupt { + pins = "gpio47"; + function = "gpio"; + input; + bias-disable; + }; + quartz_reset { + pins = "gpio21"; + function = "gpio"; + output-low; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio46", "gpio47", "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + /* POWER_LED, TP-Link */ + led_pins: led_pins { + led_power { + pins = "gpio42"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + /* BUTTON, TP-Link */ + button_pins: button_pins { + reset_button { + pins = "gpio50"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + usb_mux_sel_pins: usb_mux_pins { + mux { + pins = "gpio27"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + pcie0_pins: pcie_pins { + pcie0_rst { + pins = "gpio58"; + function = "pcie0_rst"; + drive-strength = <8>; + bias-pull-down; + }; + pcie0_wake { + pins = "gpio59"; + function = "pcie0_wake"; + drive-strength = <8>; + bias-pull-down; + }; + }; + +}; + +&soc { + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1>; + compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; + phy0: ethernet-phy@0 { + reg = <0>; + }; + phy1: ethernet-phy@1 { + reg = <1>; + }; + phy2: ethernet-phy@2 { + reg = <2>; + }; + phy3: ethernet-phy@3 { + reg = <3>; + }; + phy4: ethernet-phy@4 { + reg = <28>; + }; + phy5: ethernet-phy@5 { + reg = <4>; + }; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x3e>; /* lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + qcom,port_phyinfo { + port@0 { + port_id = <1>; + phy_address = <0>; + }; + port@1 { + port_id = <2>; + phy_address = <1>; + }; + port@2 { + port_id = <3>; + phy_address = <2>; + }; + port@3 { + port_id = <4>; + phy_address = <3>; + }; + port@4 { + port_id = <5>; + phy_address = <28>; + port_mac_sel = "QGMAC_PORT"; + }; + port@5 { + port_id = <6>; + phy_address = <4>; + }; + }; + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <4>; + phy-mode = "sgmii"; + }; + + /* POWER LED, TP-Link */ + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + led_power: led_power { + label = "blue:power"; + gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; + default-state = "on"; + }; + }; + + /* BUTTON, TP-Link */ + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset"; + linux,code = ; + gpios = <&tlmm 50 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + }; + }; + + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x18>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&spi_4 { /* BLSP1 QUP3 */ + pinctrl-0 = <&spi_3_pins>; + pinctrl-names = "default"; + cs-select = <2>; + quartz-reset-gpio = <&tlmm 21 1>; + status = "disabled"; + spidev3: spi@3 { + compatible = "qca,spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; +}; + +&serial_blsp2 { + status = "disabled"; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + status = "disabled"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; + +&ledc { + status = "disabled"; +}; + +&pcie0 { + status = "ok"; +}; + +&pcie1 { + status = "disabled"; +}; + diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts new file mode 100644 index 000000000..f8addc97f --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts @@ -0,0 +1,754 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "TP-Link EX447"; + compatible = "tplink,ex447", "qcom,ipq807x"; + qcom,msm-id = <0x143 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ + init=/init"; + bootargs-append = " swiotlb=1"; + led-boot = &led_power; + led-failsafe = &led_power; + led-running = &led_power; + led-upgrade = &led_power; + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio64"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio65"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + mux_2 { + pins = "gpio66"; + function = "pta1_0"; + drive-strength = <6>; + bias-pull-down; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_2 { + pins = "gpio25"; + function = "gpio"; + bias-pull-up; + }; + mux_3 { + pins = "gpio37"; + function = "gpio"; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_3_pins: spi_3_pins { + mux { + pins = "gpio52", "gpio53"; + function = "blsp3_spi"; + drive-strength = <8>; + bias-disable; + }; + spi_cs { + pins = "gpio22"; + function = "blsp3_spi2"; + drive-strength = <8>; + bias-disable; + }; + quartz_interrupt { + pins = "gpio47"; + function = "gpio"; + input; + bias-disable; + }; + quartz_reset { + pins = "gpio21"; + function = "gpio"; + output-low; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio46", "gpio47", "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + /* POWER_LED, TP-Link */ + led_pins: led_pins { + led_power { + pins = "gpio42"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + /* BUTTON, TP-Link */ + button_pins: button_pins { + reset_button { + pins = "gpio50"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + usb_mux_sel_pins: usb_mux_pins { + mux { + pins = "gpio27"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + pcie0_pins: pcie_pins { + pcie0_rst { + pins = "gpio58"; + function = "pcie0_rst"; + drive-strength = <8>; + bias-pull-down; + }; + pcie0_wake { + pins = "gpio59"; + function = "pcie0_wake"; + drive-strength = <8>; + bias-pull-down; + }; + }; + +}; + +&soc { + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1>; + compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; + phy0: ethernet-phy@0 { + reg = <0>; + }; + phy1: ethernet-phy@1 { + reg = <1>; + }; + phy2: ethernet-phy@2 { + reg = <2>; + }; + phy3: ethernet-phy@3 { + reg = <3>; + }; + phy4: ethernet-phy@4 { + reg = <28>; + }; + phy5: ethernet-phy@5 { + reg = <4>; + }; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x3e>; /* lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + qcom,port_phyinfo { + port@0 { + port_id = <1>; + phy_address = <0>; + }; + port@1 { + port_id = <2>; + phy_address = <1>; + }; + port@2 { + port_id = <3>; + phy_address = <2>; + }; + port@3 { + port_id = <4>; + phy_address = <3>; + }; + port@4 { + port_id = <5>; + phy_address = <28>; + port_mac_sel = "QGMAC_PORT"; + }; + port@5 { + port_id = <6>; + phy_address = <4>; + }; + }; + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; + + /* POWER LED, TP-Link */ + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + led_power: led_power { + label = "power:blue"; + gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; + default-state = "on"; + }; + }; + + /* BUTTON, TP-Link */ + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset"; + linux,code = ; + gpios = <&tlmm 50 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + }; + }; + + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x18>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&spi_4 { /* BLSP1 QUP3 */ + pinctrl-0 = <&spi_3_pins>; + pinctrl-names = "default"; + cs-select = <2>; + quartz-reset-gpio = <&tlmm 21 1>; + status = "disabled"; + spidev3: spi@3 { + compatible = "qca,spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; +}; + +&serial_blsp2 { + status = "disabled"; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + status = "disabled"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; + +&ledc { + status = "disabled"; +}; + +&pcie0 { + status = "ok"; +}; + +&pcie1 { + status = "disabled"; +}; + diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts new file mode 100644 index 000000000..0db1a5c91 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts @@ -0,0 +1,751 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-audio.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "CIG WF194C"; + compatible = "cig,wf194c", "qcom,ipq807x"; + qcom,msm-id = <0x143 0x0>, <0x158 0x0>, <0x186 0x0>, <0x188 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + sdhc1 = &sdhc_1; /* SDC1 eMMC slot */ + sdhc2 = &sdhc_2; /* SDC2 SD slot */ + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw init=/init"; + #ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; + #else + bootargs-append = " swiotlb=1 coherent_pool=2M"; + #endif + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio34"; + function = "gpio"; + drive-strength = <6>; + bias-pull-up; + output-high; + }; + mux_1 { + pins = "gpio62"; + function = "gpio"; + drive-strength = <6>; + bias-pull-up; + output-high; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + i2c_0_pins: i2c_0_pinmux { + mux { + pins = "gpio42", "gpio43"; + function = "blsp1_i2c"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + data_8 { + pins = "gpio16"; + function = "qpic_pad8"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", + "gpio9", "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + sd_pins: sd_pins { + mux { + pins = "gpio63"; + function = "sd_card"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + leds_pins: leds_pinmux { + led1_r { + pins = "gpio54"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led1_g { + pins = "gpio55"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_r { + pins = "gpio56"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_g { + pins = "gpio64"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + }; + + button_pins: button_pins { + wps_button { + pins = "gpio67"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + uniphy_pins: uniphy_pinmux { + mux_2 { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_3 { + pins = "gpio44"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + pwm_pins: pwm_pinmux { + mux_1 { + pins = "gpio25"; + function = "pwm02"; + drive-strength = <8>; + }; + }; +}; + +&soc { + pwm { + pinctrl-0 = <&pwm_pins>; + pinctrl-names = "default"; + used-pwm-indices = <1>, <0>, <0>, <0>; + status = "disabled"; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset"; + linux,code = ; + gpios = <&tlmm 67 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + status = "ok"; + + led@54 { + label = "wf194c:red:lan"; + gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + led@55 { + label = "wf194c:green:lan"; + gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>; + default-state = "on"; + }; + led@56 { + label = "wf194c:red:wan"; + gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + led@64 { + label = "wf194c:green:wan"; + gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; + default-state = "on"; + }; + }; + mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0 &tlmm 44 0>; + phy0: ethernet-phy@0 { + reg = <0xf>; /*<0>*/ + }; + phy1: ethernet-phy@1 { + reg = <0xf>; + }; + phy2: ethernet-phy@2 { + reg = <0xf>; + }; + phy3: ethernet-phy@3 { + reg = <3>; + }; + phy4: ethernet-phy@4 { + reg = <4>; + }; + phy5: ethernet-phy@5 { + compatible ="ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + }; + + ess-switch@3a000000 { + pinctrl-0 = <&uniphy_pins>; + pinctrl-names = "default"; + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x30>; /*..0x3e lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + group@1 { + sp = <61>; + cfg = <1 33 1 33>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + ucast_loop_pri = <16>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a007000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&serial_blsp2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&msm_imem { + status = "enabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + pinctrl-0 = <&i2c_0_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&sdhc_1 { + qcom,clk-rates = <400000 25000000 50000000 100000000 \ + 192000000 384000000>; + qcom,bus-speed-mode = "HS400_1p8v", "HS200_1p8v", "DDR_1p8v"; + qcom,nonremovable; + status = "ok"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&sdhc_2 { + qcom,clk-rates = <400000 25000000 50000000 100000000 \ + 192000000>; + qcom,bus-speed-mode = "HS200_1p8v", "DDR_1p8v"; + pinctrl-0 = <&sd_pins>; + pinctrl-names = "default"; + cd-gpios = <&tlmm 63 1>; + sd-ldo-gpios = <&tlmm 21 0>; + vqmmc-supply = <&ldo11>; + status = "ok"; +}; + +&qpic_lcd { + status = "ok"; +}; + +&qpic_lcd_panel { + status = "ok"; +}; diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts new file mode 100644 index 000000000..4c633fea5 --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts @@ -0,0 +1,942 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "CIG WF194c4"; + compatible = "cig,wf194c4", "qcom,ipq807x"; + qcom,msm-id = <0x156 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + /* ethernet2 = "/soc/dp3"; + ethernet3 = "/soc/dp4"; + ethernet4 = "/soc/dp5"; + ethernet5 = "/soc/dp6"; + */ + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ + init=/init"; + #ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; + #else + bootargs-append = " swiotlb=1 coherent_pool=2M"; + #endif + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { +/* + mux_0 { + pins = "gpio64"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio65"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + mux_2 { + pins = "gpio66"; + function = "pta1_0"; + drive-strength = <6>; + bias-pull-down; + }; + mux_3 { + pins = "gpio54"; + function = "pta2_0"; + drive-strength = <6>; + bias-pull-down; + }; + mux_4 { + pins = "gpio55"; + function = "pta2_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_5 { + pins = "gpio56"; + function = "pta2_2"; + drive-strength = <6>; + bias-pull-down; + }; +*/ + mux_0 { + pins = "gpio34"; + function = "gpio"; + drive-strength = <6>; + bias-pull-up; + output-high; + }; + mux_1 { + pins = "gpio62"; + function = "gpio"; + drive-strength = <6>; + bias-pull-up; + output-high; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + /*spi_3_pins: spi_3_pins { + mux { + pins = "gpio50", "gpio52", "gpio53"; + function = "blsp3_spi"; + drive-strength = <8>; + bias-disable; + }; + spi_cs { + pins = "gpio22"; + function = "blsp3_spi2"; + drive-strength = <8>; + bias-disable; + }; + quartz_interrupt { + pins = "gpio47"; + function = "gpio"; + input; + bias-disable; + }; + quartz_reset { + pins = "gpio21"; + function = "gpio"; + output-low; + bias-disable; + }; + };*/ + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + button_pins: button_pins { + wps_button { + pins = "gpio67"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + leds_pins: leds_pinmux { + led1_r { + pins = "gpio54"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led1_g { + pins = "gpio55"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_r { + pins = "gpio56"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + led2_g { + pins = "gpio64"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + }; + + /*usb_mux_sel_pins: usb_mux_pins { + mux { + pins = "gpio27"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + pcie0_pins: pcie_pins { + pcie0_rst { + pins = "gpio58"; + function = "pcie0_rst"; + drive-strength = <8>; + bias-pull-down; + }; + pcie0_wake { + pins = "gpio59"; + function = "pcie0_wake"; + drive-strength = <8>; + bias-pull-down; + }; + };*/ + uniphy_pins: uniphy_pinmux { + mux_2 { + pins = "gpio37"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_3 { + pins = "gpio44"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + +}; + +&soc { + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset"; + linux,code = ; + gpios = <&tlmm 67 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&leds_pins>; + pinctrl-names = "default"; + status = "ok"; + + led@54 { + label = "red:lan"; + gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led1_r"; + default-state = "off"; + }; + led@55 { + label = "green:lan"; + gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led1_g"; + default-state = "off"; + }; + led@56 { + label = "red:wan"; + gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led2_r"; + default-state = "off"; + }; + led@64 { + label = "green:wan"; + gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "led2_g"; + default-state = "off"; + }; + }; + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 37 0 &tlmm 44 0>; + phy0: ethernet-phy@0 { + reg = <0x10>; /*<0>*/ + }; + phy1: ethernet-phy@1 { + reg = <0x11>; + }; + phy2: ethernet-phy@2 { + reg = <0x12>; + }; + phy3: ethernet-phy@3 { + reg = <0x13>; + }; + phy4: ethernet-phy@4 { + reg = <0x14>; + }; + phy5: ethernet-phy@5 { + compatible ="ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + }; + + ess-switch@3a000000 { + pinctrl-0 = <&uniphy_pins>; + pinctrl-names = "default"; + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x32>; /*..0x3e lan port bitmap */ + switch_wan_bmp = <0x40>; /* wan port bitmap */ + switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + /*qcom,port_phyinfo { + port@0 { + port_id = <1>; + phy_address = <0>; + }; + port@1 { + port_id = <2>; + phy_address = <1>; + }; + port@2 { + port_id = <3>; + phy_address = <2>; + }; + port@3 { + port_id = <4>; + phy_address = <3>; + }; + port@4 { + port_id = <5>; + phy_address = <24>; + port_mac_sel = "QGMAC_PORT"; + }; + port@5 { + port_id = <6>; + phy_address = <28>; + port_mac_sel = "QGMAC_PORT"; + }; + };*/ + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; +/* + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <1>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <2>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <1>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <2>; + phy-mode = "sgmii"; + }; + + dp4 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <3>; + phy-mode = "sgmii"; + }; + + dp5 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a003000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <24>; + phy-mode = "sgmii"; + }; + + dp6 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a007000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; +*/ + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + // qcom,id = <1>; + // reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0x13>; + phy-mode = "sgmii"; + }; + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a007000 0x3fff>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; +/* + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + led_2g { + label = "led_2g"; + gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led_5g { + label = "led_5g"; + gpio = <&tlmm 43 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x18>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; +*/ +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; +/* +&spi_4 { + pinctrl-0 = <&spi_3_pins>; + pinctrl-names = "default"; + cs-select = <2>; + quartz-reset-gpio = <&tlmm 21 1>; + status = "ok"; + spidev3: spi@3 { + compatible = "qca,spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; +};*/ + +&serial_blsp2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&ssphy_1 { + status = "ok"; +}; + +&qusb_phy_1 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&usb3_1 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + status = "disabled"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; + +&ledc { + status = "disabled"; +}; + +&pcie0 { + status = "disabled"; +}; + +&pcie1 { + status = "disabled"; +}; + diff --git a/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf196.dts b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf196.dts new file mode 100644 index 000000000..96029697c --- /dev/null +++ b/feeds/ipq807x/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf196.dts @@ -0,0 +1,1059 @@ +/dts-v1/; +/* + * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include "qcom-ipq807x-soc.dtsi" +#include "qcom-ipq807x-audio.dtsi" +#include "qcom-ipq807x-hk-cpu.dtsi" + +/ { + #address-cells = <0x2>; + #size-cells = <0x2>; + model = "CIG WF196"; + compatible = "cig,wf196", "qcom,ipq807x"; + qcom,msm-id = <0x187 0x0>; + interrupt-parent = <&intc>; + qcom,board-id = <0x8 0x0>; + qcom,pmic-id = <0x0 0x0 0x0 0x0>; + + aliases { + /* + * Aliases as required by u-boot + * to patch MAC addresses + */ + ethernet0 = "/soc/dp1"; + ethernet1 = "/soc/dp2"; + ethernet2 = "/soc/dp3"; + ethernet3 = "/soc/dp4"; + ethernet4 = "/soc/dp5"; + ethernet5 = "/soc/dp6"; + }; + + chosen { + bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw init=/init"; + #ifdef __IPQ_MEM_PROFILE_256_MB__ + bootargs-append = " swiotlb=1"; + #else + bootargs-append = " swiotlb=1 coherent_pool=2M"; + #endif + }; + + reserved-memory { +/* No Pine attach in 256M profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) +#ifdef __IPQ_MEM_PROFILE_512_MB__ + /* 512 MB Profile + * +========+==============+=========================+ + * | Region | Start Offset | Size | + * +--------+--------------+-------------------------+ + * | NSS | 0x40000000 | 16MB | + * +--------+--------------+-------------------------+ + * | Linux | 0x41000000 | Depends on total memory | + * +--------+--------------+-------------------------+ + * | TZ APP | 0x4A400000 | 1MB | + * +--------+--------------+-------------------------+ + * | uboot | 0x4A600000 | 4MB | + * +--------+--------------+-------------------------+ + * | SBL | 0x4AA00000 | 1MB | + * +--------+--------------+-------------------------+ + * | smem | 0x4AB00000 | 1MB | + * +--------+--------------+-------------------------+ + * | TZ | 0x4AC00000 | 4MB | + * +--------+--------------+-------------------------+ + * | Q6 | 0x4B000000 | 55MB | + * +--------+--------------+-------------------------+ + * | QDSS | 0x4E700000 | 1MB | + * +--------+--------------+-------------------------+ + * | M3 Dump| 0x4E800000 | 1MB | + * +--------+--------------+-------------------------+ + * | Pine0 | 0x4E900000 | 30MB | + * +--------+--------------+-------------------------+ + * | MHI0 | 0x50700000 | 16MB | + * +--------+--------------+-------------------------+ + * | | + * | Rest of the memory for Linux | + * | | + * +=================================================+ + */ + /delete-node/ wifi_dump@4e900000; + qcn9000_pcie0@4e900000 { + no-map; + reg = <0x0 0x4e900000 0x0 0x01e00000>; + }; + + mhi_region0: dma_pool0@50700000 { + compatible = "shared-dma-pool"; + no-map; + reg = <0x0 0x50700000 0x0 0x01000000>; + }; +#else + /* Default Profile + * +========+==============+=========================+ + * | Region | Start Offset | Size | + * +--------+--------------+-------------------------+ + * | NSS | 0x40000000 | 16MB | + * +--------+--------------+-------------------------+ + * | Linux | 0x41000000 | Depends on total memory | + * +--------+--------------+-------------------------+ + * | TZ APP | 0x4A400000 | 2MB | + * +--------+--------------+-------------------------+ + * | uboot | 0x4A600000 | 4MB | + * +--------+--------------+-------------------------+ + * | SBL | 0x4AA00000 | 1MB | + * +--------+--------------+-------------------------+ + * | smem | 0x4AB00000 | 1MB | + * +--------+--------------+-------------------------+ + * | TZ | 0x4AC00000 | 4MB | + * +--------+--------------+-------------------------+ + * | Q6 | 0x4B000000 | 95MB | + * +--------+--------------+-------------------------+ + * | QDSS | 0x50F00000 | 1MB | + * +--------+--------------+-------------------------+ + * | M3 Dump| 0x51000000 | 1MB | + * +--------+--------------+-------------------------+ + * | Pine0 | 0x51100000 | 45MB | + * +--------+--------------+-------------------------+ + * | MHI0 | 0x53E00000 | 24MB | + * +--------+--------------+-------------------------+ + * | | + * | Rest of the memory for Linux | + * | | + * +=================================================+ + */ + qcn9000_pcie0@51100000 { + no-map; + reg = <0x0 0x51100000 0x0 0x02D00000>; + }; + /delete-node/ wifi_dump@51100000; + /delete-node/ wigig_dump@51300000; + + mhi_region0: dma_pool0@53e00000 { + compatible = "shared-dma-pool"; + no-map; + reg = <0x0 0x53E00000 0x0 0x01800000>; + }; +#endif +#endif + }; +}; + +&tlmm { + pinctrl-0 = <&btcoex_pins>; + pinctrl-names = "default"; + + btcoex_pins: btcoex_pins { + mux_0 { + pins = "gpio64"; + function = "pta1_1"; + drive-strength = <6>; + bias-pull-down; + }; + mux_1 { + pins = "gpio65"; + function = "pta1_2"; + drive-strength = <6>; + bias-pull-down; + }; + mux_2 { + pins = "gpio66"; + function = "pta1_0"; + drive-strength = <6>; + bias-pull-down; + }; + }; + + mdio_pins: mdio_pinmux { + mux_0 { + pins = "gpio68"; + function = "mdc"; + drive-strength = <8>; + bias-pull-up; + }; + mux_1 { + pins = "gpio69"; + function = "mdio"; + drive-strength = <8>; + bias-pull-up; + }; + mux_3 { + pins = "gpio44"; + function = "gpio"; + bias-pull-up; + }; + }; + + uart_pins: uart_pins { + mux { + pins = "gpio23", "gpio24"; + function = "blsp4_uart1"; + drive-strength = <8>; + bias-disable; + }; + }; + i2c_5_pins: i2c_5_pinmux { + mux { + pins = "gpio0", "gpio2"; + function = "blsp5_i2c"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_0_pins: spi_0_pins { + mux { + pins = "gpio38", "gpio39", "gpio40", "gpio41"; + function = "blsp0_spi"; + drive-strength = <8>; + bias-disable; + }; + }; + + spi_3_pins: spi_3_pins { + mux { + pins = "gpio50", "gpio52", "gpio53"; + function = "blsp3_spi"; + drive-strength = <8>; + bias-disable; + }; + spi_cs { + pins = "gpio22"; + function = "blsp3_spi2"; + drive-strength = <8>; + bias-disable; + }; + quartz_interrupt { + pins = "gpio47"; + function = "gpio"; + input; + bias-disable; + }; + quartz_reset { + pins = "gpio21"; + function = "gpio"; + output-low; + bias-disable; + }; + }; + + qpic_pins: qpic_pins { + data_0 { + pins = "gpio15"; + function = "qpic_pad0"; + drive-strength = <8>; + bias-disable; + }; + data_1 { + pins = "gpio12"; + function = "qpic_pad1"; + drive-strength = <8>; + bias-disable; + }; + data_2 { + pins = "gpio13"; + function = "qpic_pad2"; + drive-strength = <8>; + bias-disable; + }; + data_3 { + pins = "gpio14"; + function = "qpic_pad3"; + drive-strength = <8>; + bias-disable; + }; + data_4 { + pins = "gpio5"; + function = "qpic_pad4"; + drive-strength = <8>; + bias-disable; + }; + data_5 { + pins = "gpio6"; + function = "qpic_pad5"; + drive-strength = <8>; + bias-disable; + }; + data_6 { + pins = "gpio7"; + function = "qpic_pad6"; + drive-strength = <8>; + bias-disable; + }; + data_7 { + pins = "gpio8"; + function = "qpic_pad7"; + drive-strength = <8>; + bias-disable; + }; + qpic_pad { + pins = "gpio1", "gpio3", "gpio4", + "gpio10", "gpio11", "gpio17"; + function = "qpic_pad"; + drive-strength = <8>; + bias-disable; + }; + }; + + hsuart_pins: hsuart_pins { + mux { + pins = "gpio46", "gpio47", "gpio48", "gpio49"; + function = "blsp2_uart"; + drive-strength = <8>; + bias-disable; + }; + }; + + button_pins: button_pins { + + wps_button { + pins = "gpio67"; + function = "gpio"; + drive-strength = <8>; + bias-pull-up; + }; + }; + + led_pins: led_pins { + led_2g { + pins = "gpio42"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + + led_5g { + pins = "gpio43"; + function = "gpio"; + drive-strength = <8>; + bias-pull-down; + }; + }; + + pcie0_pins: pcie_pins { + pcie0_rst { + pins = "gpio58"; + function = "pcie0_rst"; + drive-strength = <8>; + bias-pull-down; + }; + pcie0_wake { + pins = "gpio59"; + function = "pcie0_wake"; + drive-strength = <8>; + bias-pull-down; + }; + }; + pwm_pins: pwm_pinmux { + mux_1 { + pins = "gpio25"; + function = "pwm02"; + drive-strength = <8>; + }; + mux_2 { + pins = "gpio26"; + function = "pwm12"; + drive-strength = <8>; + }; + mux_3 { + pins = "gpio27"; + function = "pwm22"; + drive-strength = <8>; + }; + }; + +}; + +&soc { + pwm { + pinctrl-0 = <&pwm_pins>; + pinctrl-names = "default"; + used-pwm-indices = <1>, <1>, <1>, <0>; + status = "ok"; + }; + gpio_keys { + compatible = "gpio-keys"; + pinctrl-0 = <&button_pins>; + pinctrl-names = "default"; + + button@1 { + label = "reset"; + linux,code = ; + gpios = <&tlmm 67 GPIO_ACTIVE_LOW>; + linux,input-type = <1>; + debounce-interval = <60>; + }; + }; + + mdio: mdio@90000 { + pinctrl-0 = <&mdio_pins>; + pinctrl-names = "default"; + phy-reset-gpio = <&tlmm 64 0 &tlmm 66 0>; + compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; + phy0: ethernet-phy@0 { + reg = <16>; + }; + phy1: ethernet-phy@1 { + reg = <17>; + }; + phy2: ethernet-phy@2 { + reg = <18>; + }; + phy3: ethernet-phy@3 { + reg = <19>; + }; + phy4: ethernet-phy@4 { + compatible ="ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + phy5: ethernet-phy@5 { + reg = <28>; + }; + }; + + ess-switch@3a000000 { + switch_cpu_bmp = <0x1>; /* cpu port bitmap */ + switch_lan_bmp = <0x3e>; /* lan port bitmap */ + switch_wan_bmp = <0xc0>; /* wan port bitmap */ + switch_mac_mode = <0xff>; /* mac mode for uniphy instance0*/ + switch_mac_mode1 = <0xd>; /* mac mode for uniphy instance1*/ + switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ + bm_tick_mode = <0>; /* bm tick mode */ + tm_tick_mode = <0>; /* tm tick mode */ + qcom,port_phyinfo { + port@0 { + port_id = <1>; + phy_address = <16>; + }; + port@1 { + port_id = <2>; + phy_address = <17>; + }; + port@2 { + port_id = <3>; + phy_address = <18>; + }; + port@3 { + port_id = <4>; + phy_address = <19>; + }; + port@4 { + port_id = <5>; + phy_address = <0>; + port_mac_sel = "QGMAC_PORT"; + ethernet-phy-ieee802.3-c45; + }; + port@5 { + port_id = <6>; + phy_address = <28>; + port_mac_sel = "QGMAC_PORT"; + }; + }; + port_scheduler_resource { + port@0 { + port_id = <0>; + ucast_queue = <0 143>; + mcast_queue = <256 271>; + l0sp = <0 35>; + l0cdrr = <0 47>; + l0edrr = <0 47>; + l1cdrr = <0 7>; + l1edrr = <0 7>; + }; + port@1 { + port_id = <1>; + ucast_queue = <144 159>; + mcast_queue = <272 275>; + l0sp = <36 39>; + l0cdrr = <48 63>; + l0edrr = <48 63>; + l1cdrr = <8 11>; + l1edrr = <8 11>; + }; + port@2 { + port_id = <2>; + ucast_queue = <160 175>; + mcast_queue = <276 279>; + l0sp = <40 43>; + l0cdrr = <64 79>; + l0edrr = <64 79>; + l1cdrr = <12 15>; + l1edrr = <12 15>; + }; + port@3 { + port_id = <3>; + ucast_queue = <176 191>; + mcast_queue = <280 283>; + l0sp = <44 47>; + l0cdrr = <80 95>; + l0edrr = <80 95>; + l1cdrr = <16 19>; + l1edrr = <16 19>; + }; + port@4 { + port_id = <4>; + ucast_queue = <192 207>; + mcast_queue = <284 287>; + l0sp = <48 51>; + l0cdrr = <96 111>; + l0edrr = <96 111>; + l1cdrr = <20 23>; + l1edrr = <20 23>; + }; + port@5 { + port_id = <5>; + ucast_queue = <208 223>; + mcast_queue = <288 291>; + l0sp = <52 55>; + l0cdrr = <112 127>; + l0edrr = <112 127>; + l1cdrr = <24 27>; + l1edrr = <24 27>; + }; + port@6 { + port_id = <6>; + ucast_queue = <224 239>; + mcast_queue = <292 295>; + l0sp = <56 59>; + l0cdrr = <128 143>; + l0edrr = <128 143>; + l1cdrr = <28 31>; + l1edrr = <28 31>; + }; + port@7 { + port_id = <7>; + ucast_queue = <240 255>; + mcast_queue = <296 299>; + l0sp = <60 63>; + l0cdrr = <144 159>; + l0edrr = <144 159>; + l1cdrr = <32 35>; + l1edrr = <32 35>; + }; + }; + port_scheduler_config { + port@0 { + port_id = <0>; + l1scheduler { + group@0 { + sp = <0 1>; /*L0 SPs*/ + /*cpri cdrr epri edrr*/ + cfg = <0 0 0 0>; + }; + }; + l0scheduler { + group@0 { + /*unicast queues*/ + ucast_queue = <0 4 8>; + /*multicast queues*/ + mcast_queue = <256 260>; + /*sp cpri cdrr epri edrr*/ + cfg = <0 0 0 0 0>; + }; + group@1 { + ucast_queue = <1 5 9>; + mcast_queue = <257 261>; + cfg = <0 1 1 1 1>; + }; + group@2 { + ucast_queue = <2 6 10>; + mcast_queue = <258 262>; + cfg = <0 2 2 2 2>; + }; + group@3 { + ucast_queue = <3 7 11>; + mcast_queue = <259 263>; + cfg = <0 3 3 3 3>; + }; + }; + }; + port@1 { + port_id = <1>; + l1scheduler { + group@0 { + sp = <36>; + cfg = <0 8 0 8>; + }; + group@1 { + sp = <37>; + cfg = <1 9 1 9>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <144>; + ucast_loop_pri = <16>; + mcast_queue = <272>; + mcast_loop_pri = <4>; + cfg = <36 0 48 0 48>; + }; + }; + }; + port@2 { + port_id = <2>; + l1scheduler { + group@0 { + sp = <40>; + cfg = <0 12 0 12>; + }; + group@1 { + sp = <41>; + cfg = <1 13 1 13>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <160>; + ucast_loop_pri = <16>; + mcast_queue = <276>; + mcast_loop_pri = <4>; + cfg = <40 0 64 0 64>; + }; + }; + }; + port@3 { + port_id = <3>; + l1scheduler { + group@0 { + sp = <44>; + cfg = <0 16 0 16>; + }; + group@1 { + sp = <45>; + cfg = <1 17 1 17>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <176>; + ucast_loop_pri = <16>; + mcast_queue = <280>; + mcast_loop_pri = <4>; + cfg = <44 0 80 0 80>; + }; + }; + }; + port@4 { + port_id = <4>; + l1scheduler { + group@0 { + sp = <48>; + cfg = <0 20 0 20>; + }; + group@1 { + sp = <49>; + cfg = <1 21 1 21>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <192>; + ucast_loop_pri = <16>; + mcast_queue = <284>; + mcast_loop_pri = <4>; + cfg = <48 0 96 0 96>; + }; + }; + }; + port@5 { + port_id = <5>; + l1scheduler { + group@0 { + sp = <52>; + cfg = <0 24 0 24>; + }; + group@1 { + sp = <53>; + cfg = <1 25 1 25>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <208>; + ucast_loop_pri = <16>; + mcast_queue = <288>; + mcast_loop_pri = <4>; + cfg = <52 0 112 0 112>; + }; + }; + }; + port@6 { + port_id = <6>; + l1scheduler { + group@0 { + sp = <56>; + cfg = <0 28 0 28>; + }; + group@1 { + sp = <57>; + cfg = <1 29 1 29>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <224>; + ucast_loop_pri = <16>; + mcast_queue = <292>; + mcast_loop_pri = <4>; + cfg = <56 0 128 0 128>; + }; + }; + }; + port@7 { + port_id = <7>; + l1scheduler { + group@0 { + sp = <60>; + cfg = <0 32 0 32>; + }; + group@1 { + sp = <61>; + cfg = <1 33 1 33>; + }; + }; + l0scheduler { + group@0 { + ucast_queue = <240>; + ucast_loop_pri = <16>; + mcast_queue = <296>; + cfg = <60 0 144 0 144>; + }; + }; + }; + }; + }; + + dp1 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <1>; + reg = <0x3a001000 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <16>; + phy-mode = "sgmii"; + }; + + dp2 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <2>; + reg = <0x3a001200 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <17>; + phy-mode = "sgmii"; + }; + + dp3 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <3>; + reg = <0x3a001400 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <18>; + phy-mode = "sgmii"; + }; + + dp4 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <4>; + reg = <0x3a001600 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <19>; + phy-mode = "sgmii"; + }; + + dp5 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <5>; + reg = <0x3a001800 0x200>; + qcom,mactype = <1>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <0>; + phy-mode = "sgmii"; + }; + + dp6 { + device_type = "network"; + compatible = "qcom,nss-dp"; + qcom,id = <6>; + reg = <0x3a001a00 0x200>; + qcom,mactype = <0>; + local-mac-address = [000000000000]; + qcom,link-poll = <1>; + qcom,phy-mdio-addr = <28>; + phy-mode = "sgmii"; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + led_2g { + label = "led_2g"; + gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led_5g { + label = "led_5g"; + gpio = <&tlmm 43 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + nss-macsec0 { + compatible = "qcom,nss-macsec"; + phy_addr = <0>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + nss-macsec1 { + compatible = "qcom,nss-macsec"; + phy_addr = <0x1c>; + phy_access_mode = <0>; + mdiobus = <&mdio>; + }; + i2c_5: i2c@78ba000 { + compatible = "qcom,i2c-qup-v2.2.1"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x78ba000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_AHB_CLK>, + <&gcc GCC_BLSP1_QUP6_I2C_APPS_CLK>; + clock-names = "iface", "core"; + clock-frequency = <100000>; + dmas = <&blsp_dma 23>, <&blsp_dma 22>; + dma-names = "rx", "tx"; + }; +}; + +&serial_blsp4 { + pinctrl-0 = <&uart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&spi_0 { /* BLSP1 QUP1 */ + pinctrl-0 = <&spi_0_pins>; + pinctrl-names = "default"; + cs-select = <0>; + status = "ok"; + + m25p80@0 { + #address-cells = <1>; + #size-cells = <1>; + reg = <0>; + compatible = "n25q128a11"; + linux,modalias = "m25p80", "n25q128a11"; + spi-max-frequency = <50000000>; + use-default-sizes; + }; +}; + +&serial_blsp2 { + pinctrl-0 = <&hsuart_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&msm_imem { + status = "disabled"; +}; + +&ssphy_0 { + status = "ok"; +}; + +&qusb_phy_0 { + status = "ok"; +}; + +&usb3_0 { + status = "ok"; +}; + +&cryptobam { + status = "ok"; +}; + +&crypto { + status = "ok"; +}; + +&i2c_0 { + status = "disabled"; +}; + +&i2c_1 { + status = "disabled"; +}; + +&i2c_5 { + pinctrl-0 = <&i2c_5_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_bam { + status = "ok"; +}; + +&nand { + pinctrl-0 = <&qpic_pins>; + pinctrl-names = "default"; + status = "ok"; +}; + +&qpic_lcd { + status = "disabled"; +}; + +&qpic_lcd_panel { + status = "disabled"; +}; + +&pcie0 { + status = "ok"; +}; + + +&pcie0_rp { + status = "ok"; + + mhi_0: qcom,mhi@0 { + reg = <0 0 0 0 0 >; + qrtr_instance_id = <0x20>; + #address-cells = <0x2>; + #size-cells = <0x2>; +#if !defined(__IPQ_MEM_PROFILE_256_MB__) + memory-region = <&mhi_region0>; +#endif + +#if !defined(__CNSS2__) + base-addr = <0x51100000>; + m3-dump-addr = <0x53400000>; + etr-addr = <0x53500000>; + qcom,caldb-addr = <0x53600000>; + mhi,max-channels = <30>; + mhi,timeout = <10000>; + qcom,board_id= <0xa4>; + + pcie0_mhi: pcie0_mhi { + status = "ok"; + }; +#endif + }; +}; + +&pcie1_rp { + status = "disabled"; + + mhi_1: qcom,mhi@1 { + pcie1_mhi: pcie1_mhi { + status = "disabled"; + }; + }; +}; + +&npu_cpr { + status = "disabled"; +}; + +&nss0 { + npu-supply = <&dummy_reg>; + mx-supply = <&dummy_reg>; +}; + +#ifndef __IPQ_MEM_PROFILE_256_MB__ +&nss1 { + npu-supply = <&dummy_reg>; + mx-supply = <&dummy_reg>; +}; +#endif + +&wifi1 { + qcom,board_id = <0x294>; + status = "ok"; +}; + +/* No support for QCN9000 in 256M profile */ +#if !defined(__IPQ_MEM_PROFILE_256_MB__) +&wifi2 { +#ifdef __IPQ_MEM_PROFILE_512_MB__ + /* QCN9000 tgt-mem-mode=1 layout - 30MB + * +=========+==============+=========+ + * | Region | Start Offset | Size | + * +---------+--------------+---------+ + * | HREMOTE | 0x4E900000 | 20MB | + * +---------+--------------+---------+ + * | M3 Dump | 0x4FD00000 | 1MB | + * +---------+--------------+---------+ + * | ETR | 0x4FE00000 | 1MB | + * +---------+--------------+---------+ + * | Caldb | 0x4FF00000 | 8MB | + * +==================================+ + */ + base-addr = <0x4E900000>; + m3-dump-addr = <0x4FD00000>; + etr-addr = <0x4FE00000>; + caldb-addr = <0x4FF00000>; + hremote-size = <0x1400000>; + tgt-mem-mode = <0x1>; +#else + /* QCN9000 tgt-mem-mode=0 layout - 45MB + * +=========+==============+=========+ + * | Region | Start Offset | Size | + * +---------+--------------+---------+ + * | HREMOTE | 0x51100000 | 35MB | + * +---------+--------------+---------+ + * | M3 Dump | 0x53400000 | 1MB | + * +---------+--------------+---------+ + * | ETR | 0x53500000 | 1MB | + * +---------+--------------+---------+ + * | Caldb | 0x53600000 | 8MB | + * +==================================+ + */ + base-addr = <0x51100000>; + m3-dump-addr = <0x53400000>; + etr-addr = <0x53500000>; + caldb-addr = <0x53600000>; + hremote-size = <0x2300000>; + tgt-mem-mode = <0x0>; +#endif + caldb-size = <0x800000>; + board_id = <0xa4>; + status = "ok"; +#if defined(__CNSS2__) + pcie0_mhi: pcie0_mhi { + status = "ok"; + }; +#endif +}; +&wifi3 { + status = "disabled"; +}; + +#include "qcom-ipq807x-mhi.dtsi" +#endif diff --git a/feeds/ipq807x/ipq807x/image/Makefile b/feeds/ipq807x/ipq807x/image/Makefile new file mode 100644 index 000000000..3a219e6d6 --- /dev/null +++ b/feeds/ipq807x/ipq807x/image/Makefile @@ -0,0 +1,26 @@ +include $(TOPDIR)/rules.mk +include $(INCLUDE_DIR)/image.mk + +IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(IMG_PREFIX_VERNUM)$(IMG_PREFIX_VERCODE)$(IMG_PREFIX_EXTRA)$(BOARD) + +# default all platform image(fit) build +define Device/Default + PROFILES = Default $$(DEVICE_NAME) + KERNEL_NAME := zImage + FILESYSTEMS := squashfs + DEVICE_DTS_DIR := $(DTS_DIR) + KERNEL_IN_UBI := 1 + ROOTFSNAME_IN_UBI := ubi_rootfs + BLOCKSIZE := 128k + PAGESIZE := 2048 + IMAGES := sysupgrade.tar nand-factory.bin + IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata + IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand + KERNEL_NAME := Image + KERNEL = kernel-bin | gzip | fit gzip $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb + KERNEL_INITRAMFS = kernel-bin | gzip | fit gzip $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb +endef + +include $(SUBTARGET).mk + +$(eval $(call BuildImage)) diff --git a/feeds/ipq807x/ipq807x/image/ipq50xx.mk b/feeds/ipq807x/ipq807x/image/ipq50xx.mk new file mode 100644 index 000000000..f20d54190 --- /dev/null +++ b/feeds/ipq807x/ipq807x/image/ipq50xx.mk @@ -0,0 +1,10 @@ +KERNEL_LOADADDR := 0x41208000 + +define Device/qcom_mp03_3 + DEVICE_TITLE := Qualcomm Maple 03.3 + DEVICE_DTS := qcom-ipq5018-mp03.3 + SUPPORTED_DEVICES := qcom,ipq5018-mp03.3 + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq5018 + DEVICE_DTS_CONFIG := config@mp03.3 +endef +TARGET_DEVICES += qcom_mp03_3 diff --git a/feeds/ipq807x/ipq807x/image/ipq60xx.mk b/feeds/ipq807x/ipq807x/image/ipq60xx.mk new file mode 100644 index 000000000..ecea4da00 --- /dev/null +++ b/feeds/ipq807x/ipq807x/image/ipq60xx.mk @@ -0,0 +1,74 @@ +KERNEL_LOADADDR := 0x41008000 + +define Device/cig_wf188 + DEVICE_TITLE := Cigtech WF-188 + DEVICE_DTS := qcom-ipq6018-cig-wf188 + DEVICE_DTS_CONFIG := config@cp03-c1 + SUPPORTED_DEVICES := cig,wf188 + IMAGES := sysupgrade.tar + IMAGE/sysupgrade.tar/squashfs := append-rootfs | pad-rootfs | sysupgrade-tar rootfs=$$$$@ | append-metadata + DEVICE_PACKAGES := ath11k-wifi-cig-wf188 uboot-env +endef +TARGET_DEVICES += cig_wf188 + +define Device/cig_wf188n + DEVICE_TITLE := Cigtech WF-188n + DEVICE_DTS := qcom-ipq6018-cig-wf188n + DEVICE_DTS_CONFIG := config@cp03-c1 + SUPPORTED_DEVICES := cig,wf188n + DEVICE_PACKAGES := ath11k-wifi-cig-wf188n uboot-env +endef +TARGET_DEVICES += cig_wf188n + +define Device/hfcl_ion4xe + DEVICE_TITLE := HFCL ION4Xe + DEVICE_DTS := qcom-ipq6018-hfcl-ion4xe + DEVICE_DTS_CONFIG := config@cp01-c1 + SUPPORTED_DEVICES := hfcl,ion4xe + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 uboot-envtools +endef +TARGET_DEVICES += hfcl_ion4xe + +define Device/hfcl_ion4xi + DEVICE_TITLE := HFCL ION4Xi + DEVICE_DTS := qcom-ipq6018-hfcl-ion4xi + DEVICE_DTS_CONFIG := config@cp01-c1 + SUPPORTED_DEVICES := hfcl,ion4xi + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 uboot-envtools +endef +TARGET_DEVICES += hfcl_ion4xi + +define Device/edgecore_eap101 + DEVICE_TITLE := EdgeCore EAP101 + DEVICE_DTS := qcom-ipq6018-edgecore-eap101 + DEVICE_DTS_CONFIG := config@cp01-c1 + SUPPORTED_DEVICES := edgecore,eap101 + DEVICE_PACKAGES := ath11k-wifi-edgecore-eap101 uboot-envtools +endef +TARGET_DEVICES += edgecore_eap101 + +define Device/wallys_dr6018 + DEVICE_TITLE := Wallys DR6018 + DEVICE_DTS := qcom-ipq6018-wallys-dr6018 + DEVICE_DTS_CONFIG := config@cp01-c4 + SUPPORTED_DEVICES := wallys,dr6018 + DEVICE_PACKAGES := ath11k-wifi-wallys-dr6018 uboot-envtools +endef +TARGET_DEVICES += wallys_dr6018 + +define Device/wallys_dr6018_v4 + DEVICE_TITLE := Wallys DR6018 V4 + DEVICE_DTS := qcom-ipq6018-wallys-dr6018-v4 + DEVICE_DTS_CONFIG := config@cp01-c4 + SUPPORTED_DEVICES := wallys,dr6018-v4 + DEVICE_PACKAGES := ath11k-wifi-wallys-dr6018-v4 uboot-envtools +endef +TARGET_DEVICES += wallys_dr6018_v4 + +define Device/qcom_cp01_c1 + DEVICE_TITLE := Qualcomm Cypress C1 + DEVICE_DTS := qcom-ipq6018-cp01-c1 + SUPPORTED_DEVICES := qcom,ipq6018-cp01 + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 +endef +TARGET_DEVICES += qcom_cp01_c1 diff --git a/feeds/ipq807x/ipq807x/image/ipq807x.mk b/feeds/ipq807x/ipq807x/image/ipq807x.mk new file mode 100644 index 000000000..3d36673b7 --- /dev/null +++ b/feeds/ipq807x/ipq807x/image/ipq807x.mk @@ -0,0 +1,91 @@ +KERNEL_LOADADDR := 0x41208000 + +define Device/qcom_hk01 + DEVICE_TITLE := Qualcomm Hawkeye HK01 + DEVICE_DTS := qcom-ipq807x-hk01 + DEVICE_DTS_CONFIG=config@hk01 + SUPPORTED_DEVICES := qcom,ipq807x-hk01 + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq8074 +endef +TARGET_DEVICES += qcom_hk01 + +define Device/qcom_hk14 + DEVICE_TITLE := Qualcomm Hawkeye HK14 + DEVICE_DTS := qcom-ipq807x-hk14 + DEVICE_DTS_CONFIG=config@hk14 + SUPPORTED_DEVICES := qcom,ipq807x-hk14 + DEVICE_PACKAGES := ath11k-wifi-qcom-ipq8074 kmod-ath11k-pci ath11k-firmware-qcn9000 +endef +TARGET_DEVICES += qcom_hk14 + +define Device/cig_wf194 + DEVICE_TITLE := CIG WF194C + DEVICE_DTS := qcom-ipq807x-wf194c + DEVICE_DTS_CONFIG=config@hk01 + SUPPORTED_DEVICES := cig,wf194c + DEVICE_PACKAGES := ath11k-wifi-cig-wf194c aq-fw-download uboot-envtools kmod-usb3 kmod-usb2 +endef +TARGET_DEVICES += cig_wf194 + +define Device/cig_wf194c4 + DEVICE_TITLE := CIG WF194C4 + DEVICE_DTS := qcom-ipq807x-wf194c4 + DEVICE_DTS_CONFIG=config@hk09 + SUPPORTED_DEVICES := cig,wf194c4 + DEVICE_PACKAGES := ath11k-wifi-cig-wf194c4 aq-fw-download uboot-envtools kmod-usb3 kmod-usb2 +endef +TARGET_DEVICES += cig_wf194c4 + +define Device/cig_wf196 + DEVICE_TITLE := CIG WF196 + DEVICE_DTS := qcom-ipq807x-wf196 + DEVICE_DTS_CONFIG=config@hk14 + SUPPORTED_DEVICES := cig,wf196 + DEVICE_PACKAGES := ath11k-wifi-cig-wf196 aq-fw-download uboot-envtools kmod-usb3 kmod-usb2 \ + ath11k-firmware-qcn9000 ath11k-wifi-cig-wf196_6g +endef +TARGET_DEVICES += cig_wf196 + +define Device/edgecore_eap102 + DEVICE_TITLE := Edgecore EAP102 + DEVICE_DTS := qcom-ipq807x-eap102 + DEVICE_DTS_CONFIG=config@ac02 + SUPPORTED_DEVICES := edgecore,eap102 + DEVICE_PACKAGES := ath11k-wifi-edgecore-eap102 kmod-usb2 kmod-usb3 uboot-envtools +endef +TARGET_DEVICES += edgecore_eap102 + +define Device/edgecore_eap106 + DEVICE_TITLE := Edgecore EAP106 + DEVICE_DTS := qcom-ipq807x-eap106 + DEVICE_DTS_CONFIG=config@hk02 + SUPPORTED_DEVICES := edgecore,eap106 + DEVICE_PACKAGES := ath11k-wifi-edgecore-eap106 aq-fw-download kmod-usb2 kmod-usb3 uboot-envtools +endef +TARGET_DEVICES += edgecore_eap106 + +define Device/tplink_ex227 + DEVICE_TITLE := TP-Link EX227 + DEVICE_DTS := qcom-ipq807x-ex227 + DEVICE_DTS_CONFIG=config@hk07 + SUPPORTED_DEVICES := tplink,ex227 + DEVICE_PACKAGES := ath11k-wifi-tplink-ex227 + IMAGES := sysupgrade.tar nand-factory.bin nand-factory.ubi + IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata + IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand + IMAGE/nand-factory.ubi := append-ubi +endef +TARGET_DEVICES += tplink_ex227 + +define Device/tplink_ex447 + DEVICE_TITLE := TP-Link EX447 + DEVICE_DTS := qcom-ipq807x-ex447 + DEVICE_DTS_CONFIG=config@hk09 + SUPPORTED_DEVICES := tplink,ex447 + DEVICE_PACKAGES := ath11k-wifi-tplink-ex447 + IMAGES := sysupgrade.tar nand-factory.bin nand-factory.ubi + IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata + IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand + IMAGE/nand-factory.ubi := append-ubi +endef +TARGET_DEVICES += tplink_ex447 diff --git a/feeds/ipq807x/ipq807x/ipq50xx/config-default b/feeds/ipq807x/ipq807x/ipq50xx/config-default new file mode 100644 index 000000000..b8e202c87 --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq50xx/config-default @@ -0,0 +1,84 @@ +# CONFIG_AHCI_IPQ is not set +CONFIG_ARCH_IPQ5018=y +# CONFIG_DIAGFWD_BRIDGE_CODE is not set +CONFIG_IPQ_ADSS_5018=y +CONFIG_IPQ_APSS_5018=y +CONFIG_IPQ_GCC_5018=y +# CONFIG_NET_SWITCHDEV is not set +CONFIG_NUM_ALT_PARTITION=16 +CONFIG_PINCTRL_IPQ5018=y +# CONFIG_IPC_LOGGING is not set +CONFIG_IPQ_SUBSYSTEM_DUMP=y +CONFIG_SPS=y +CONFIG_SPS_SUPPORT_NDP_BAM=y +CONFIG_CORESIGHT=y +CONFIG_CORESIGHT_CSR=y +CONFIG_CORESIGHT_CTI=y +CONFIG_CORESIGHT_EVENT=y +CONFIG_CORESIGHT_HWEVENT=y +CONFIG_CORESIGHT_LINKS_AND_SINKS=y +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y +CONFIG_CORESIGHT_QCOM_REPLICATOR=y +# CONFIG_INPUT_PM8941_PWRKEY is not set +CONFIG_MDIO_QCA=y +# CONFIG_CRYPTO_ALL_CASES is not set +CONFIG_CRYPTO_DEV_QCOM_ICE=y +# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SHA512=y +# CONFIG_CORESIGHT_QPDI is not set +# CONFIG_CORESIGHT_SINK_ETBV10 is not set +CONFIG_CORESIGHT_SINK_TPIU=y +# CONFIG_CORESIGHT_SOURCE_DUMMY is not set +CONFIG_CORESIGHT_SOURCE_ETM3X=y +CONFIG_CORESIGHT_SOURCE_ETM4X=y +# CONFIG_CORESIGHT_REMOTE_ETM is not set +CONFIG_CORESIGHT_STM=y +CONFIG_CORESIGHT_TPDA=y +CONFIG_CORESIGHT_TPDM=y +# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set +CONFIG_IIO=y +# CONFIG_IIO_BUFFER is not set +# CONFIG_IIO_TRIGGER is not set +CONFIG_PCIE_DW_PLAT=y +CONFIG_PHY_IPQ_UNIPHY_PCIE=y +CONFIG_VMSPLIT_2G=y +# CONFIG_VMSPLIT_3G is not set +CONFIG_PPS=y +CONFIG_PTP_1588_CLOCK=y +# CONFIG_DP83640_PHY is not set +CONFIG_PWM_IPQ5018=y +CONFIG_QCOM_APM=y +CONFIG_QCOM_DCC=y +# CONFIG_QCOM_SPMI_TEMP_ALARM is not set +CONFIG_MMC_SDHCI_MSM_ICE=y +CONFIG_USB_BAM=y +CONFIG_MAILBOX=y +# CONFIG_USB_QCOM_DIAG_BRIDGE is not set +# CONFIG_USB_CONFIGFS_F_DIAG is not set +# CONFIG_NF_IPV6_DUMMY_HEADER is not set +CONFIG_RMNET_DATA=y +CONFIG_RMNET_DATA_DEBUG_PKT=y +CONFIG_MTD_NAND_SERIAL=y +CONFIG_PAGE_SCOPE_MULTI_PAGE_READ=y +# CONFIG_RMNET_DATA_FC is not set +CONFIG_CRYPTO_NO_ZERO_LEN_HASH=y +CONFIG_CRYPTO_DISABLE_AES192_TEST=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set +# CONFIG_CPU_FREQ_GOV_PERFORMANCE is not set +# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set +# CONFIG_CPU_FREQ_GOV_ONDEMAND is not set +# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set +CONFIG_QTI_EUD=y +CONFIG_USB_QCA_M31_PHY=y +CONFIG_QGIC2_MSI=y +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y +CONFIG_PWM_IPQ4019=y +CONFIG_RMNET=y +CONFIG_QCOM_QMI_RMNET=y +CONFIG_QCOM_QMI_DFC=y +CONFIG_QCOM_QMI_POWER_COLLAPSE=y +CONFIG_RMNET_CTL=y +CONFIG_RMNET_CTL_DEBUG=y +CONFIG_SND_SOC_IPQ_LPASS=y +CONFIG_SND_SOC_IPQ_LPASS_PCM_RAW=y +# CONFIG_SND_SOC_IPQ_PCM_RAW is not set diff --git a/feeds/ipq807x/ipq807x/ipq50xx/config-lowmem b/feeds/ipq807x/ipq807x/ipq50xx/config-lowmem new file mode 100644 index 000000000..b1b817ef6 --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq50xx/config-lowmem @@ -0,0 +1,73 @@ +# CONFIG_AHCI_IPQ is not set +CONFIG_ARCH_IPQ5018=y +# CONFIG_DIAGFWD_BRIDGE_CODE is not set +CONFIG_IPQ_ADSS_5018=y +CONFIG_IPQ_APSS_5018=y +CONFIG_IPQ_GCC_5018=y +# CONFIG_NET_SWITCHDEV is not set +CONFIG_NUM_ALT_PARTITION=16 +CONFIG_PINCTRL_IPQ5018=y +# CONFIG_IPC_LOGGING is not set +CONFIG_IPQ_SUBSYSTEM_DUMP=y +# CONFIG_SPS is not set +# CONFIG_SPS_SUPPORT_NDP_BAM is not set +# CONFIG_CORESIGHT is not set +# CONFIG_INPUT_PM8941_PWRKEY is not set +CONFIG_MDIO_QCA=y +# CONFIG_CRYPTO_ALL_CASES is not set +# CONFIG_CRYPTO_DEV_QCOM_ICE is not set +# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +# CONFIG_CRYPTO_SHA512 is not set +# CONFIG_IIO is not set +# CONFIG_IIO_BUFFER is not set +# CONFIG_IIO_TRIGGER is not set +CONFIG_PCIE_DW_PLAT=y +CONFIG_PHY_IPQ_UNIPHY_PCIE=y +CONFIG_VMSPLIT_2G=y +# CONFIG_VMSPLIT_3G is not set +# CONFIG_PPS is not set +# CONFIG_PTP_1588_CLOCK is not set +# CONFIG_DP83640_PHY is not set +CONFIG_PWM_IPQ5018=y +CONFIG_QCOM_APM=y +# CONFIG_QCOM_DCC is not set +# CONFIG_QCOM_SPMI_TEMP_ALARM is not set +CONFIG_MMC_SDHCI_MSM_ICE=y +CONFIG_USB_BAM=y +CONFIG_MAILBOX=y +# CONFIG_USB_QCOM_DIAG_BRIDGE is not set +# CONFIG_USB_CONFIGFS_F_DIAG is not set +# CONFIG_NF_IPV6_DUMMY_HEADER is not set +# CONFIG_RMNET_DATA is not set +# CONFIG_RMNET_DATA_DEBUG_PKT is not set +CONFIG_MTD_NAND_SERIAL=y +CONFIG_PAGE_SCOPE_MULTI_PAGE_READ=y +# CONFIG_RMNET_DATA_FC is not set +# CONFIG_CRYPTO_NO_ZERO_LEN_HASH is not set +# CONFIG_CRYPTO_DISABLE_AES192_TEST is not set +# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set +# CONFIG_CPU_FREQ_GOV_PERFORMANCE is not set +# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set +# CONFIG_CPU_FREQ_GOV_ONDEMAND is not set +# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set +CONFIG_QTI_EUD=y +CONFIG_USB_QCA_M31_PHY=y +CONFIG_SQUASHFS_XZ=y +# CONFIG_SQUASHFS_ZLIB is not set +# CONFIG_JFFS2_LZMA is not set +CONFIG_JFFS2_ZLIB=y +# CONFIG_LZO_COMPRESS is not set +# CONFIG_LZO_DECOMPRESS is not set +CONFIG_XZ_DEC=y +# CONFIG_XZ_DEC_X86 is not set +# CONFIG_XZ_DEC_POWERPC is not set +# CONFIG_XZ_DEC_IA64 is not set +CONFIG_XZ_DEC_ARM=y +# CONFIG_XZ_DEC_ARMTHUMB is not set +# CONFIG_XZ_DEC_SPARC is not set +CONFIG_XZ_DEC_BCJ=y +# CONFIG_LZO_COMPRESS is not set +# CONFIG_LZO_DECOMPRESS is not set +# CONFIG_CRYPTO is not set +CONFIG_QGIC2_MSI=y +CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y diff --git a/feeds/ipq807x/ipq807x/ipq50xx/target.mk b/feeds/ipq807x/ipq807x/ipq50xx/target.mk new file mode 100644 index 000000000..649f398ba --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq50xx/target.mk @@ -0,0 +1,10 @@ + +SUBTARGET:=ipq50xx +BOARDNAME:=IPQ50XX +CPU_TYPE:=cortex-a7 + +DEFAULT_PACKAGES += ath11k-firmware-ipq50xx qca-nss-fw-ipq50xx + +define Target/Description + Build firmware image for IPQ50xx SoC devices. +endef diff --git a/feeds/ipq807x/ipq807x/ipq60xx/config-default b/feeds/ipq807x/ipq807x/ipq60xx/config-default new file mode 100644 index 000000000..3d7c59698 --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq60xx/config-default @@ -0,0 +1,122 @@ +CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS=y +CONFIG_ANDROID=y +# CONFIG_ANDROID_BINDER_IPC is not set +# CONFIG_AQ_PHY is not set +CONFIG_ARCH_HIBERNATION_POSSIBLE=y +CONFIG_ARCH_IPQ6018=y +# CONFIG_ARCH_MSM8X60 is not set +CONFIG_ARM_DMA_IOMMU_ALIGNMENT=8 +CONFIG_ARM_DMA_USE_IOMMU=y +CONFIG_ARM_HEAVY_MB=y +CONFIG_ARM_QTI_IPQ60XX_CPUFREQ=y +CONFIG_ARM_SMMU=y +CONFIG_ASN1=y +CONFIG_ASSOCIATIVE_ARRAY=y +CONFIG_CACHE_L2X0=y +CONFIG_CLZ_TAB=y +CONFIG_CMA=y +CONFIG_CMA_ALIGNMENT=8 +CONFIG_CMA_AREAS=7 +CONFIG_CMA_DEBUG=y +CONFIG_CMA_DEBUGFS=y +CONFIG_CMA_SIZE_MBYTES=0 +CONFIG_CMA_SIZE_SEL_MBYTES=y +CONFIG_CORESIGHT=y +CONFIG_CORESIGHT_CSR=y +CONFIG_CORESIGHT_CTI=y +CONFIG_CORESIGHT_HWEVENT=y +CONFIG_CORESIGHT_LINKS_AND_SINKS=y +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y +CONFIG_CORESIGHT_QCOM_REPLICATOR=y +CONFIG_CORESIGHT_SINK_TPIU=y +CONFIG_CORESIGHT_SOURCE_ETM3X=y +CONFIG_CORESIGHT_SOURCE_ETM4X=y +CONFIG_CORESIGHT_STM=y +CONFIG_CORESIGHT_TPDA=y +CONFIG_CORESIGHT_TPDM=y +CONFIG_CRC_CCITT=m +CONFIG_CRYPTO_AKCIPHER=y +CONFIG_CRYPTO_AKCIPHER2=y +CONFIG_CRYPTO_DRBG=y +CONFIG_CRYPTO_DRBG_HMAC=y +CONFIG_CRYPTO_DRBG_MENU=y +CONFIG_CRYPTO_GF128MUL=y +CONFIG_CRYPTO_GHASH=y +CONFIG_CRYPTO_HASH=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_JITTERENTROPY=y +# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_NULL2=y +CONFIG_CRYPTO_PCOMP2=y +CONFIG_CRYPTO_RNG_DEFAULT=y +CONFIG_CRYPTO_SHA512=y +CONFIG_DMA_CMA=y +CONFIG_DMA_SHARED_BUFFER=y +CONFIG_DT_IDLE_STATES=y +CONFIG_EDAC_ATOMIC_SCRUB=y +CONFIG_EDAC_SUPPORT=y +CONFIG_EXTCON=y +CONFIG_EXTCON_USB_GPIO=y +CONFIG_FB_DEFERRED_IO=y +CONFIG_FREEZER=y +CONFIG_GPIO_WATCHDOG=y +CONFIG_GPIO_WATCHDOG_ARCH_INITCALL=y +CONFIG_IOMMU_API=y +CONFIG_IOMMU_DMA=y +CONFIG_IOMMU_IOVA=y +CONFIG_IOMMU_IO_PGTABLE=y +# CONFIG_IOMMU_IO_PGTABLE_ARMV7S is not set +CONFIG_IOMMU_IO_PGTABLE_LPAE=y +# CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST is not set +CONFIG_IOMMU_SUPPORT=y +CONFIG_ION=y +CONFIG_ION_MSM=y +CONFIG_IPQ_APSS_6018=y +CONFIG_IPQ_GCC_6018=y +CONFIG_IPQ_MEM_PROFILE=256 +CONFIG_KASAN_SHADOW_OFFSET=0x5f000000 +CONFIG_LEDS_PCA9956B=y +CONFIG_MDIO_QCA=y +CONFIG_MEMORY_ISOLATION=y +CONFIG_MIGRATION=y +CONFIG_MPILIB=y +CONFIG_MSM_SECURE_BUFFER=y +CONFIG_NEED_SG_DMA_LENGTH=y +CONFIG_NET_SWITCHDEV=y +CONFIG_NUM_ALT_PARTITION=16 +CONFIG_OF_IOMMU=y +CONFIG_OID_REGISTRY=y +CONFIG_OUTER_CACHE=y +CONFIG_OUTER_CACHE_SYNC=y +CONFIG_PAGE_OFFSET=0x80000000 +CONFIG_PINCTRL_IPQ6018=y +# CONFIG_PKCS7_MESSAGE_PARSER is not set +# CONFIG_PL310_ERRATA_588369 is not set +# CONFIG_PL310_ERRATA_727915 is not set +# CONFIG_PL310_ERRATA_753970 is not set +# CONFIG_PL310_ERRATA_769419 is not set +CONFIG_PPS=y +CONFIG_PTP_1588_CLOCK=y +CONFIG_PWM_IPQ4019=y +CONFIG_QCOM_APM=y +CONFIG_QCOM_DCC=y +CONFIG_QCOM_QMI_HELPERS=y +CONFIG_REGMAP_I2C=y +CONFIG_REGMAP_SPMI=y +CONFIG_REGULATOR_CPR3=y +CONFIG_REGULATOR_CPR3_NPU=y +CONFIG_REGULATOR_CPR4_APSS=y +# CONFIG_SKB_FIXED_SIZE_2K is not set +CONFIG_SOC_BUS=y +CONFIG_SPS=y +CONFIG_SPS_SUPPORT_NDP_BAM=y +CONFIG_STAGING=y +CONFIG_SUSPEND_FREEZER=y +# CONFIG_USB_GADGET is not set +CONFIG_USB_OHCI_LITTLE_ENDIAN=y +# CONFIG_VFIO is not set +CONFIG_VIRTIO=y +CONFIG_VIRTUALIZATION=y +CONFIG_VMSPLIT_2G=y +# CONFIG_VMSPLIT_3G is not set +CONFIG_WANT_DEV_COREDUMP=y diff --git a/feeds/ipq807x/ipq807x/ipq60xx/profiles/default.mk b/feeds/ipq807x/ipq807x/ipq60xx/profiles/default.mk new file mode 100644 index 000000000..f47e73acf --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq60xx/profiles/default.mk @@ -0,0 +1,9 @@ +define Profile/Default + NAME:=Default Profile (minimum package set) +endef + +define Profile/Default/Description +Default package set compatible with most boards. +endef +$(eval $(call Profile,Default)) + diff --git a/feeds/ipq807x/ipq807x/ipq60xx/target.mk b/feeds/ipq807x/ipq807x/ipq60xx/target.mk new file mode 100644 index 000000000..cf2bf7b9c --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq60xx/target.mk @@ -0,0 +1,8 @@ +SUBTARGET:=ipq60xx +BOARDNAME:=IPQ60xx based boards + +DEFAULT_PACKAGES += ath11k-firmware-ipq60xx qca-nss-fw-ipq60xx + +define Target/Description + Build images for IPQ60xx systems. +endef diff --git a/feeds/ipq807x/ipq807x/ipq807x/config-default b/feeds/ipq807x/ipq807x/ipq807x/config-default new file mode 100644 index 000000000..f1e8aadc9 --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq807x/config-default @@ -0,0 +1,78 @@ +# CONFIG_AHCI_IPQ is not set +CONFIG_ARCH_IPQ807x=y +# CONFIG_DIAGFWD_BRIDGE_CODE is not set +CONFIG_IPQ_ADSS_807x=y +CONFIG_IPQ_APSS_807x=y +CONFIG_IPQ_GCC_807x=y +CONFIG_NET_SWITCHDEV=y +CONFIG_NUM_ALT_PARTITION=16 +CONFIG_PINCTRL_IPQ807x=y +# CONFIG_IPC_LOGGING is not set +CONFIG_IPQ_SUBSYSTEM_DUMP=y +CONFIG_SPS=y +CONFIG_SPS_SUPPORT_NDP_BAM=y +CONFIG_CORESIGHT=y +CONFIG_CORESIGHT_CSR=y +CONFIG_CORESIGHT_CTI=y +CONFIG_CORESIGHT_EVENT=y +CONFIG_CORESIGHT_HWEVENT=y +CONFIG_CORESIGHT_LINKS_AND_SINKS=y +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y +CONFIG_CORESIGHT_QCOM_REPLICATOR=y +CONFIG_CORESIGHT_STREAM=m +# CONFIG_INPUT_PM8941_PWRKEY is not set +CONFIG_MDIO_QCA=y +# CONFIG_CRYPTO_ALL_CASES is not set +CONFIG_CRYPTO_DEV_QCOM_ICE=y +# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SHA512=y +# CONFIG_CORESIGHT_QPDI is not set +# CONFIG_CORESIGHT_SINK_ETBV10 is not set +CONFIG_CORESIGHT_SINK_TPIU=y +# CONFIG_CORESIGHT_SOURCE_DUMMY is not set +CONFIG_CORESIGHT_SOURCE_ETM3X=y +CONFIG_CORESIGHT_SOURCE_ETM4X=y +# CONFIG_CORESIGHT_REMOTE_ETM is not set +CONFIG_CORESIGHT_STM=y +CONFIG_CORESIGHT_TPDA=y +CONFIG_CORESIGHT_TPDM=y +CONFIG_AQUANTIA_PHY=y +# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set +CONFIG_IIO=y +# CONFIG_IIO_BUFFER is not set +# CONFIG_IIO_TRIGGER is not set +CONFIG_PCIE_DW_PLAT=y +CONFIG_VMSPLIT_2G=y +# CONFIG_VMSPLIT_3G is not set +CONFIG_PPS=y +CONFIG_PTP_1588_CLOCK=y +# CONFIG_DP83640_PHY is not set +CONFIG_PWM_IPQ4019=y +CONFIG_QCOM_APM=y +CONFIG_QCOM_DCC=y +# CONFIG_QCOM_SPMI_TEMP_ALARM is not set +CONFIG_QCOM_SPMI_VADC=y +CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS=y +CONFIG_REGULATOR_CPR3=y +CONFIG_REGULATOR_CPR3_NPU=y +CONFIG_REGULATOR_CPR4_APSS=y +CONFIG_MMC_SDHCI_MSM_ICE=y +CONFIG_USB_BAM=y +CONFIG_USB_QCOM_KS_BRIDGE=m +CONFIG_MAILBOX=y +# CONFIG_USB_QCOM_DIAG_BRIDGE is not set +# CONFIG_USB_CONFIGFS_F_DIAG is not set +# CONFIG_NF_IPV6_DUMMY_HEADER is not set +CONFIG_RMNET=y +CONFIG_RMNET_DATA=y +CONFIG_RMNET_DATA_DEBUG_PKT=y +# CONFIG_RMNET_DATA_FC is not set +CONFIG_QCOM_QMI_RMNET=y +CONFIG_QCOM_QMI_DFC=y +CONFIG_QCOM_QMI_POWER_COLLAPSE=y +CONFIG_RMNET_CTL=y +CONFIG_RMNET_CTL_DEBUG=y +CONFIG_MHI_BUS_TEST=y +CONFIG_MHI_DEBUG=y +CONFIG_MHI_NETDEV=y +CONFIG_MHI_UCI=y diff --git a/feeds/ipq807x/ipq807x/ipq807x/profiles/default.mk b/feeds/ipq807x/ipq807x/ipq807x/profiles/default.mk new file mode 100644 index 000000000..f47e73acf --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq807x/profiles/default.mk @@ -0,0 +1,9 @@ +define Profile/Default + NAME:=Default Profile (minimum package set) +endef + +define Profile/Default/Description +Default package set compatible with most boards. +endef +$(eval $(call Profile,Default)) + diff --git a/feeds/ipq807x/ipq807x/ipq807x/target.mk b/feeds/ipq807x/ipq807x/ipq807x/target.mk new file mode 100644 index 000000000..7c24b66e6 --- /dev/null +++ b/feeds/ipq807x/ipq807x/ipq807x/target.mk @@ -0,0 +1,7 @@ +SUBTARGET:=ipq807x +BOARDNAME:=IPQ807x based boards + +DEFAULT_PACKAGES += ath11k-firmware-ipq807x qca-nss-fw-ipq807x +define Target/Description + Build images for IPQ807x systems. +endef diff --git a/feeds/ipq807x/ipq807x/modules.mk b/feeds/ipq807x/ipq807x/modules.mk new file mode 100644 index 000000000..0223ff259 --- /dev/null +++ b/feeds/ipq807x/ipq807x/modules.mk @@ -0,0 +1,61 @@ +define KernelPackage/usb-phy-ipq807x + TITLE:=DWC3 USB QCOM PHY driver for IPQ807x + DEPENDS:=@TARGET_ipq807x + KCONFIG:= \ + CONFIG_USB_QCOM_QUSB_PHY \ + CONFIG_USB_QCOM_QMP_PHY + FILES:= \ + $(LINUX_DIR)/drivers/usb/phy/phy-msm-qusb.ko \ + $(LINUX_DIR)/drivers/usb/phy/phy-msm-ssusb-qmp.ko + AUTOLOAD:=$(call AutoLoad,45,phy-msm-qusb phy-msm-ssusb-qmp,1) + $(call AddDepends/usb) +endef + +define KernelPackage/usb-phy-ipq807x/description + This driver provides support for the USB PHY drivers + within the IPQ807x SoCs. +endef + +$(eval $(call KernelPackage,usb-phy-ipq807x)) + + +define KernelPackage/qrtr_mproc + TITLE:= Ath11k Specific kernel configs for IPQ807x and IPQ60xx + DEPENDS+= @TARGET_ipq807x + KCONFIG:= \ + CONFIG_QRTR=y \ + CONFIG_QRTR_MHI=y \ + CONFIG_MHI_BUS=y \ + CONFIG_MHI_QTI=y \ + CONFIG_QCOM_APCS_IPC=y \ + CONFIG_QCOM_GLINK_SSR=y \ + CONFIG_QCOM_Q6V5_WCSS=y \ + CONFIG_MSM_RPM_RPMSG=y \ + CONFIG_RPMSG_QCOM_GLINK_RPM=y \ + CONFIG_REGULATOR_RPM_GLINK=y \ + CONFIG_QCOM_SYSMON=y \ + CONFIG_RPMSG=y \ + CONFIG_RPMSG_CHAR=y \ + CONFIG_RPMSG_QCOM_GLINK_SMEM=y \ + CONFIG_RPMSG_QCOM_SMD=y \ + CONFIG_QRTR_SMD=y \ + CONFIG_QCOM_QMI_HELPERS=y \ + CONFIG_SAMPLES=y \ + CONFIG_SAMPLE_QMI_CLIENT=m \ + CONFIG_SAMPLE_TRACE_EVENTS=n \ + CONFIG_SAMPLE_KOBJECT=n \ + CONFIG_SAMPLE_KPROBES=n \ + CONFIG_SAMPLE_KRETPROBES=n \ + CONFIG_SAMPLE_HW_BREAKPOINT=n \ + CONFIG_SAMPLE_KFIFO=n \ + CONFIG_SAMPLE_CONFIGFS=n \ + CONFIG_SAMPLE_RPMSG_CLIENT=n \ + CONFIG_MAILBOX=y \ + CONFIG_DIAG_OVER_QRTR=y +endef + +define KernelPackage/qrtr_mproc/description +Kernel configs for ath11k support specific to ipq807x and IPQ60xx +endef + +$(eval $(call KernelPackage,qrtr_mproc)) diff --git a/feeds/ipq807x/ipq807x/patches/100-qrtr-ns.patch b/feeds/ipq807x/ipq807x/patches/100-qrtr-ns.patch new file mode 100644 index 000000000..850e64477 --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/100-qrtr-ns.patch @@ -0,0 +1,976 @@ +Index: linux-4.4.60/net/qrtr/ns.c +=================================================================== +--- /dev/null ++++ linux-4.4.60/net/qrtr/ns.c +@@ -0,0 +1,760 @@ ++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause ++/* ++ * Copyright (c) 2015, Sony Mobile Communications Inc. ++ * Copyright (c) 2013, The Linux Foundation. All rights reserved. ++ * Copyright (c) 2020, Linaro Ltd. ++ */ ++ ++#include ++#include ++#include ++#include ++ ++#include "qrtr.h" ++ ++#define CREATE_TRACE_POINTS ++#include ++ ++static RADIX_TREE(nodes, GFP_KERNEL); ++ ++static struct { ++ struct socket *sock; ++ struct sockaddr_qrtr bcast_sq; ++ struct list_head lookups; ++ struct workqueue_struct *workqueue; ++ struct work_struct work; ++ int local_node; ++} qrtr_ns; ++ ++static const char * const qrtr_ctrl_pkt_strings[] = { ++ [QRTR_TYPE_HELLO] = "hello", ++ [QRTR_TYPE_BYE] = "bye", ++ [QRTR_TYPE_NEW_SERVER] = "new-server", ++ [QRTR_TYPE_DEL_SERVER] = "del-server", ++ [QRTR_TYPE_DEL_CLIENT] = "del-client", ++ [QRTR_TYPE_RESUME_TX] = "resume-tx", ++ [QRTR_TYPE_EXIT] = "exit", ++ [QRTR_TYPE_PING] = "ping", ++ [QRTR_TYPE_NEW_LOOKUP] = "new-lookup", ++ [QRTR_TYPE_DEL_LOOKUP] = "del-lookup", ++}; ++ ++struct qrtr_server_filter { ++ unsigned int service; ++ unsigned int instance; ++ unsigned int ifilter; ++}; ++ ++struct qrtr_lookup { ++ unsigned int service; ++ unsigned int instance; ++ ++ struct sockaddr_qrtr sq; ++ struct list_head li; ++}; ++ ++struct qrtr_server { ++ unsigned int service; ++ unsigned int instance; ++ ++ unsigned int node; ++ unsigned int port; ++ ++ struct list_head qli; ++}; ++ ++struct qrtr_node { ++ unsigned int id; ++ struct radix_tree_root servers; ++}; ++ ++static struct qrtr_node *node_get(unsigned int node_id) ++{ ++ struct qrtr_node *node; ++ ++ node = radix_tree_lookup(&nodes, node_id); ++ if (node) ++ return node; ++ ++ /* If node didn't exist, allocate and insert it to the tree */ ++ node = kzalloc(sizeof(*node), GFP_KERNEL); ++ if (!node) ++ return NULL; ++ ++ node->id = node_id; ++ ++ radix_tree_insert(&nodes, node_id, node); ++ ++ return node; ++} ++ ++static int server_match(const struct qrtr_server *srv, ++ const struct qrtr_server_filter *f) ++{ ++ unsigned int ifilter = f->ifilter; ++ ++ if (f->service != 0 && srv->service != f->service) ++ return 0; ++ if (!ifilter && f->instance) ++ ifilter = ~0; ++ ++ return (srv->instance & ifilter) == f->instance; ++} ++ ++static int service_announce_new(struct sockaddr_qrtr *dest, ++ struct qrtr_server *srv) ++{ ++ struct qrtr_ctrl_pkt pkt; ++ struct msghdr msg = { }; ++ struct kvec iv; ++ ++ trace_qrtr_ns_service_announce_new(srv->service, srv->instance, ++ srv->node, srv->port); ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = cpu_to_le32(QRTR_TYPE_NEW_SERVER); ++ pkt.server.service = cpu_to_le32(srv->service); ++ pkt.server.instance = cpu_to_le32(srv->instance); ++ pkt.server.node = cpu_to_le32(srv->node); ++ pkt.server.port = cpu_to_le32(srv->port); ++ ++ msg.msg_name = (struct sockaddr *)dest; ++ msg.msg_namelen = sizeof(*dest); ++ ++ return kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++} ++ ++static int service_announce_del(struct sockaddr_qrtr *dest, ++ struct qrtr_server *srv) ++{ ++ struct qrtr_ctrl_pkt pkt; ++ struct msghdr msg = { }; ++ struct kvec iv; ++ int ret; ++ ++ trace_qrtr_ns_service_announce_del(srv->service, srv->instance, ++ srv->node, srv->port); ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = cpu_to_le32(QRTR_TYPE_DEL_SERVER); ++ pkt.server.service = cpu_to_le32(srv->service); ++ pkt.server.instance = cpu_to_le32(srv->instance); ++ pkt.server.node = cpu_to_le32(srv->node); ++ pkt.server.port = cpu_to_le32(srv->port); ++ ++ msg.msg_name = (struct sockaddr *)dest; ++ msg.msg_namelen = sizeof(*dest); ++ ++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++ if (ret < 0) ++ pr_err("failed to announce del service\n"); ++ ++ return ret; ++} ++ ++static void lookup_notify(struct sockaddr_qrtr *to, struct qrtr_server *srv, ++ bool new) ++{ ++ struct qrtr_ctrl_pkt pkt; ++ struct msghdr msg = { }; ++ struct kvec iv; ++ int ret; ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = new ? cpu_to_le32(QRTR_TYPE_NEW_SERVER) : ++ cpu_to_le32(QRTR_TYPE_DEL_SERVER); ++ if (srv) { ++ pkt.server.service = cpu_to_le32(srv->service); ++ pkt.server.instance = cpu_to_le32(srv->instance); ++ pkt.server.node = cpu_to_le32(srv->node); ++ pkt.server.port = cpu_to_le32(srv->port); ++ } ++ ++ msg.msg_name = (struct sockaddr *)to; ++ msg.msg_namelen = sizeof(*to); ++ ++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++ if (ret < 0) ++ pr_err("failed to send lookup notification\n"); ++} ++ ++static int announce_servers(struct sockaddr_qrtr *sq) ++{ ++ struct radix_tree_iter iter; ++ struct qrtr_server *srv; ++ struct qrtr_node *node; ++ void __rcu **slot; ++ int ret; ++ ++ node = node_get(qrtr_ns.local_node); ++ if (!node) ++ return 0; ++ ++ /* Announce the list of servers registered in this node */ ++ radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { ++ srv = radix_tree_deref_slot(slot); ++ ++ ret = service_announce_new(sq, srv); ++ if (ret < 0) { ++ pr_err("failed to announce new service\n"); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static struct qrtr_server *server_add(unsigned int service, ++ unsigned int instance, ++ unsigned int node_id, ++ unsigned int port) ++{ ++ struct qrtr_server *srv; ++ struct qrtr_server *old; ++ struct qrtr_node *node; ++ ++ if (!service || !port) ++ return NULL; ++ ++ srv = kzalloc(sizeof(*srv), GFP_KERNEL); ++ if (!srv) ++ return NULL; ++ ++ srv->service = service; ++ srv->instance = instance; ++ srv->node = node_id; ++ srv->port = port; ++ ++ node = node_get(node_id); ++ if (!node) ++ goto err; ++ ++ /* Delete the old server on the same port */ ++ old = radix_tree_lookup(&node->servers, port); ++ if (old) { ++ radix_tree_delete(&node->servers, port); ++ kfree(old); ++ } ++ ++ radix_tree_insert(&node->servers, port, srv); ++ ++ trace_qrtr_ns_server_add(srv->service, srv->instance, ++ srv->node, srv->port); ++ ++ return srv; ++ ++err: ++ kfree(srv); ++ return NULL; ++} ++ ++static int server_del(struct qrtr_node *node, unsigned int port) ++{ ++ struct qrtr_lookup *lookup; ++ struct qrtr_server *srv; ++ struct list_head *li; ++ ++ srv = radix_tree_lookup(&node->servers, port); ++ if (!srv) ++ return -ENOENT; ++ ++ radix_tree_delete(&node->servers, port); ++ ++ /* Broadcast the removal of local servers */ ++ if (srv->node == qrtr_ns.local_node) ++ service_announce_del(&qrtr_ns.bcast_sq, srv); ++ ++ /* Announce the service's disappearance to observers */ ++ list_for_each(li, &qrtr_ns.lookups) { ++ lookup = container_of(li, struct qrtr_lookup, li); ++ if (lookup->service && lookup->service != srv->service) ++ continue; ++ if (lookup->instance && lookup->instance != srv->instance) ++ continue; ++ ++ lookup_notify(&lookup->sq, srv, false); ++ } ++ ++ kfree(srv); ++ ++ return 0; ++} ++ ++static int say_hello(struct sockaddr_qrtr *dest) ++{ ++ struct qrtr_ctrl_pkt pkt; ++ struct msghdr msg = { }; ++ struct kvec iv; ++ int ret; ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = cpu_to_le32(QRTR_TYPE_HELLO); ++ ++ msg.msg_name = (struct sockaddr *)dest; ++ msg.msg_namelen = sizeof(*dest); ++ ++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++ if (ret < 0) ++ pr_err("failed to send hello msg\n"); ++ ++ return ret; ++} ++ ++/* Announce the list of servers registered on the local node */ ++static int ctrl_cmd_hello(struct sockaddr_qrtr *sq) ++{ ++ int ret; ++ ++ ret = say_hello(sq); ++ if (ret < 0) ++ return ret; ++ ++ return announce_servers(sq); ++} ++ ++static int ctrl_cmd_bye(struct sockaddr_qrtr *from) ++{ ++ struct qrtr_node *local_node; ++ struct radix_tree_iter iter; ++ struct qrtr_ctrl_pkt pkt; ++ struct qrtr_server *srv; ++ struct sockaddr_qrtr sq; ++ struct msghdr msg = { }; ++ struct qrtr_node *node; ++ void __rcu **slot; ++ struct kvec iv; ++ int ret; ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ node = node_get(from->sq_node); ++ if (!node) ++ return 0; ++ ++ /* Advertise removal of this client to all servers of remote node */ ++ radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { ++ srv = radix_tree_deref_slot(slot); ++ server_del(node, srv->port); ++ } ++ ++ /* Advertise the removal of this client to all local servers */ ++ local_node = node_get(qrtr_ns.local_node); ++ if (!local_node) ++ return 0; ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = cpu_to_le32(QRTR_TYPE_BYE); ++ pkt.client.node = cpu_to_le32(from->sq_node); ++ ++ radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { ++ srv = radix_tree_deref_slot(slot); ++ ++ sq.sq_family = AF_QIPCRTR; ++ sq.sq_node = srv->node; ++ sq.sq_port = srv->port; ++ ++ msg.msg_name = (struct sockaddr *)&sq; ++ msg.msg_namelen = sizeof(sq); ++ ++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++ if (ret < 0) { ++ pr_err("failed to send bye cmd\n"); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, ++ unsigned int node_id, unsigned int port) ++{ ++ struct qrtr_node *local_node; ++ struct radix_tree_iter iter; ++ struct qrtr_lookup *lookup; ++ struct qrtr_ctrl_pkt pkt; ++ struct msghdr msg = { }; ++ struct qrtr_server *srv; ++ struct sockaddr_qrtr sq; ++ struct qrtr_node *node; ++ struct list_head *tmp; ++ struct list_head *li; ++ void __rcu **slot; ++ struct kvec iv; ++ int ret; ++ ++ iv.iov_base = &pkt; ++ iv.iov_len = sizeof(pkt); ++ ++ /* Don't accept spoofed messages */ ++ if (from->sq_node != node_id) ++ return -EINVAL; ++ ++ /* Local DEL_CLIENT messages comes from the port being closed */ ++ if (from->sq_node == qrtr_ns.local_node && from->sq_port != port) ++ return -EINVAL; ++ ++ /* Remove any lookups by this client */ ++ list_for_each_safe(li, tmp, &qrtr_ns.lookups) { ++ lookup = container_of(li, struct qrtr_lookup, li); ++ if (lookup->sq.sq_node != node_id) ++ continue; ++ if (lookup->sq.sq_port != port) ++ continue; ++ ++ list_del(&lookup->li); ++ kfree(lookup); ++ } ++ ++ /* Remove the server belonging to this port */ ++ node = node_get(node_id); ++ if (node) ++ server_del(node, port); ++ ++ /* Advertise the removal of this client to all local servers */ ++ local_node = node_get(qrtr_ns.local_node); ++ if (!local_node) ++ return 0; ++ ++ memset(&pkt, 0, sizeof(pkt)); ++ pkt.cmd = cpu_to_le32(QRTR_TYPE_DEL_CLIENT); ++ pkt.client.node = cpu_to_le32(node_id); ++ pkt.client.port = cpu_to_le32(port); ++ ++ radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { ++ srv = radix_tree_deref_slot(slot); ++ ++ sq.sq_family = AF_QIPCRTR; ++ sq.sq_node = srv->node; ++ sq.sq_port = srv->port; ++ ++ msg.msg_name = (struct sockaddr *)&sq; ++ msg.msg_namelen = sizeof(sq); ++ ++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); ++ if (ret < 0) { ++ pr_err("failed to send del client cmd\n"); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++static int ctrl_cmd_new_server(struct sockaddr_qrtr *from, ++ unsigned int service, unsigned int instance, ++ unsigned int node_id, unsigned int port) ++{ ++ struct qrtr_lookup *lookup; ++ struct qrtr_server *srv; ++ struct list_head *li; ++ int ret = 0; ++ ++ /* Ignore specified node and port for local servers */ ++ if (from->sq_node == qrtr_ns.local_node) { ++ node_id = from->sq_node; ++ port = from->sq_port; ++ } ++ ++ /* Don't accept spoofed messages */ ++ if (from->sq_node != node_id) ++ return -EINVAL; ++ ++ srv = server_add(service, instance, node_id, port); ++ if (!srv) ++ return -EINVAL; ++ ++ if (srv->node == qrtr_ns.local_node) { ++ ret = service_announce_new(&qrtr_ns.bcast_sq, srv); ++ if (ret < 0) { ++ pr_err("failed to announce new service\n"); ++ return ret; ++ } ++ } ++ ++ /* Notify any potential lookups about the new server */ ++ list_for_each(li, &qrtr_ns.lookups) { ++ lookup = container_of(li, struct qrtr_lookup, li); ++ if (lookup->service && lookup->service != service) ++ continue; ++ if (lookup->instance && lookup->instance != instance) ++ continue; ++ ++ lookup_notify(&lookup->sq, srv, true); ++ } ++ ++ return ret; ++} ++ ++static int ctrl_cmd_del_server(struct sockaddr_qrtr *from, ++ unsigned int service, unsigned int instance, ++ unsigned int node_id, unsigned int port) ++{ ++ struct qrtr_node *node; ++ ++ /* Ignore specified node and port for local servers*/ ++ if (from->sq_node == qrtr_ns.local_node) { ++ node_id = from->sq_node; ++ port = from->sq_port; ++ } ++ ++ /* Don't accept spoofed messages */ ++ if (from->sq_node != node_id) ++ return -EINVAL; ++ ++ /* Local servers may only unregister themselves */ ++ if (from->sq_node == qrtr_ns.local_node && from->sq_port != port) ++ return -EINVAL; ++ ++ node = node_get(node_id); ++ if (!node) ++ return -ENOENT; ++ ++ return server_del(node, port); ++} ++ ++static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, ++ unsigned int service, unsigned int instance) ++{ ++ struct radix_tree_iter node_iter; ++ struct qrtr_server_filter filter; ++ struct radix_tree_iter srv_iter; ++ struct qrtr_lookup *lookup; ++ struct qrtr_node *node; ++ void __rcu **node_slot; ++ void __rcu **srv_slot; ++ ++ /* Accept only local observers */ ++ if (from->sq_node != qrtr_ns.local_node) ++ return -EINVAL; ++ ++ lookup = kzalloc(sizeof(*lookup), GFP_KERNEL); ++ if (!lookup) ++ return -ENOMEM; ++ ++ lookup->sq = *from; ++ lookup->service = service; ++ lookup->instance = instance; ++ list_add_tail(&lookup->li, &qrtr_ns.lookups); ++ ++ memset(&filter, 0, sizeof(filter)); ++ filter.service = service; ++ filter.instance = instance; ++ ++ radix_tree_for_each_slot(node_slot, &nodes, &node_iter, 0) { ++ node = radix_tree_deref_slot(node_slot); ++ ++ radix_tree_for_each_slot(srv_slot, &node->servers, ++ &srv_iter, 0) { ++ struct qrtr_server *srv; ++ ++ srv = radix_tree_deref_slot(srv_slot); ++ if (!server_match(srv, &filter)) ++ continue; ++ ++ lookup_notify(from, srv, true); ++ } ++ } ++ ++ /* Empty notification, to indicate end of listing */ ++ lookup_notify(from, NULL, true); ++ ++ return 0; ++} ++ ++static void ctrl_cmd_del_lookup(struct sockaddr_qrtr *from, ++ unsigned int service, unsigned int instance) ++{ ++ struct qrtr_lookup *lookup; ++ struct list_head *tmp; ++ struct list_head *li; ++ ++ list_for_each_safe(li, tmp, &qrtr_ns.lookups) { ++ lookup = container_of(li, struct qrtr_lookup, li); ++ if (lookup->sq.sq_node != from->sq_node) ++ continue; ++ if (lookup->sq.sq_port != from->sq_port) ++ continue; ++ if (lookup->service != service) ++ continue; ++ if (lookup->instance && lookup->instance != instance) ++ continue; ++ ++ list_del(&lookup->li); ++ kfree(lookup); ++ } ++} ++ ++static void qrtr_ns_worker(struct work_struct *work) ++{ ++ const struct qrtr_ctrl_pkt *pkt; ++ size_t recv_buf_size = 4096; ++ struct sockaddr_qrtr sq; ++ struct msghdr msg = { }; ++ unsigned int cmd; ++ ssize_t msglen; ++ void *recv_buf; ++ struct kvec iv; ++ int ret; ++ ++ msg.msg_name = (struct sockaddr *)&sq; ++ msg.msg_namelen = sizeof(sq); ++ ++ recv_buf = kzalloc(recv_buf_size, GFP_KERNEL); ++ if (!recv_buf) ++ return; ++ ++ for (;;) { ++ iv.iov_base = recv_buf; ++ iv.iov_len = recv_buf_size; ++ ++ msglen = kernel_recvmsg(qrtr_ns.sock, &msg, &iv, 1, ++ iv.iov_len, MSG_DONTWAIT); ++ ++ if (msglen == -EAGAIN) ++ break; ++ ++ if (msglen < 0) { ++ pr_err("error receiving packet: %zd\n", msglen); ++ break; ++ } ++ ++ pkt = recv_buf; ++ cmd = le32_to_cpu(pkt->cmd); ++ if (cmd < ARRAY_SIZE(qrtr_ctrl_pkt_strings) && ++ qrtr_ctrl_pkt_strings[cmd]) ++ trace_qrtr_ns_message(qrtr_ctrl_pkt_strings[cmd], ++ sq.sq_node, sq.sq_port); ++ ++ ret = 0; ++ switch (cmd) { ++ case QRTR_TYPE_HELLO: ++ ret = ctrl_cmd_hello(&sq); ++ break; ++ case QRTR_TYPE_BYE: ++ ret = ctrl_cmd_bye(&sq); ++ break; ++ case QRTR_TYPE_DEL_CLIENT: ++ ret = ctrl_cmd_del_client(&sq, ++ le32_to_cpu(pkt->client.node), ++ le32_to_cpu(pkt->client.port)); ++ break; ++ case QRTR_TYPE_NEW_SERVER: ++ ret = ctrl_cmd_new_server(&sq, ++ le32_to_cpu(pkt->server.service), ++ le32_to_cpu(pkt->server.instance), ++ le32_to_cpu(pkt->server.node), ++ le32_to_cpu(pkt->server.port)); ++ break; ++ case QRTR_TYPE_DEL_SERVER: ++ ret = ctrl_cmd_del_server(&sq, ++ le32_to_cpu(pkt->server.service), ++ le32_to_cpu(pkt->server.instance), ++ le32_to_cpu(pkt->server.node), ++ le32_to_cpu(pkt->server.port)); ++ break; ++ case QRTR_TYPE_EXIT: ++ case QRTR_TYPE_PING: ++ case QRTR_TYPE_RESUME_TX: ++ break; ++ case QRTR_TYPE_NEW_LOOKUP: ++ ret = ctrl_cmd_new_lookup(&sq, ++ le32_to_cpu(pkt->server.service), ++ le32_to_cpu(pkt->server.instance)); ++ break; ++ case QRTR_TYPE_DEL_LOOKUP: ++ ctrl_cmd_del_lookup(&sq, ++ le32_to_cpu(pkt->server.service), ++ le32_to_cpu(pkt->server.instance)); ++ break; ++ } ++ ++ if (ret < 0) ++ pr_err("failed while handling packet from %d:%d", ++ sq.sq_node, sq.sq_port); ++ } ++ ++ kfree(recv_buf); ++} ++ ++static void qrtr_ns_data_ready(struct sock *sk) ++{ ++ queue_work(qrtr_ns.workqueue, &qrtr_ns.work); ++} ++ ++void qrtr_ns_init(void) ++{ ++ struct sockaddr_qrtr sq; ++ int sl = sizeof(sq); ++ int ret; ++ ++ INIT_LIST_HEAD(&qrtr_ns.lookups); ++ INIT_WORK(&qrtr_ns.work, qrtr_ns_worker); ++ ++ ret = sock_create_kern(&init_net, AF_QIPCRTR, SOCK_DGRAM, ++ PF_QIPCRTR, &qrtr_ns.sock); ++ if (ret < 0) ++ return; ++ ++ ret = kernel_getsockname(qrtr_ns.sock, (struct sockaddr *)&sq, &sl); ++ if (ret < 0) { ++ pr_err("failed to get socket name\n"); ++ goto err_sock; ++ } ++ ++ qrtr_ns.workqueue = alloc_workqueue("qrtr_ns_handler", WQ_UNBOUND, 1); ++ if (!qrtr_ns.workqueue) ++ goto err_sock; ++ ++ qrtr_ns.sock->sk->sk_data_ready = qrtr_ns_data_ready; ++ ++ sq.sq_port = QRTR_PORT_CTRL; ++ qrtr_ns.local_node = sq.sq_node; ++ ++ ret = kernel_bind(qrtr_ns.sock, (struct sockaddr *)&sq, sizeof(sq)); ++ if (ret < 0) { ++ pr_err("failed to bind to socket\n"); ++ goto err_wq; ++ } ++ ++ qrtr_ns.bcast_sq.sq_family = AF_QIPCRTR; ++ qrtr_ns.bcast_sq.sq_node = QRTR_NODE_BCAST; ++ qrtr_ns.bcast_sq.sq_port = QRTR_PORT_CTRL; ++ ++ ret = say_hello(&qrtr_ns.bcast_sq); ++ if (ret < 0) ++ goto err_wq; ++ ++ return; ++ ++err_wq: ++ destroy_workqueue(qrtr_ns.workqueue); ++err_sock: ++ sock_release(qrtr_ns.sock); ++} ++EXPORT_SYMBOL_GPL(qrtr_ns_init); ++ ++void qrtr_ns_remove(void) ++{ ++ cancel_work_sync(&qrtr_ns.work); ++ destroy_workqueue(qrtr_ns.workqueue); ++ sock_release(qrtr_ns.sock); ++} ++EXPORT_SYMBOL_GPL(qrtr_ns_remove); ++ ++MODULE_AUTHOR("Manivannan Sadhasivam "); ++MODULE_DESCRIPTION("Qualcomm IPC Router Nameservice"); ++MODULE_LICENSE("Dual BSD/GPL"); +Index: linux-4.4.60/net/qrtr/qrtr.c +=================================================================== +--- linux-4.4.60.orig/net/qrtr/qrtr.c ++++ linux-4.4.60/net/qrtr/qrtr.c +@@ -135,6 +135,8 @@ static DEFINE_IDR(qrtr_ports); + static DEFINE_MUTEX(qrtr_port_lock); + static DEFINE_MUTEX(qrtr_node_locking); + ++static struct delayed_work qrtr_ns_work; ++ + /** + * struct qrtr_node - endpoint node + * @ep_lock: lock for endpoint management and callbacks +@@ -1765,33 +1767,6 @@ static int qrtr_create(struct net *net, + return 0; + } + +-static const struct nla_policy qrtr_policy[IFA_MAX + 1] = { +- [IFA_LOCAL] = { .type = NLA_U32 }, +-}; +- +-static int qrtr_addr_doit(struct sk_buff *skb, struct nlmsghdr *nlh) +-{ +- struct nlattr *tb[IFA_MAX + 1]; +- struct ifaddrmsg *ifm; +- int rc; +- +- if (!netlink_capable(skb, CAP_NET_ADMIN)) +- return -EPERM; +- +- ASSERT_RTNL(); +- +- rc = nlmsg_parse(nlh, sizeof(*ifm), tb, IFA_MAX, qrtr_policy); +- if (rc < 0) +- return rc; +- +- ifm = nlmsg_data(nlh); +- if (!tb[IFA_LOCAL]) +- return -EINVAL; +- +- qrtr_local_nid = nla_get_u32(tb[IFA_LOCAL]); +- return 0; +-} +- + static const struct net_proto_family qrtr_family = { + .owner = THIS_MODULE, + .family = AF_QIPCRTR, +@@ -1811,7 +1786,8 @@ static int __init qrtr_proto_init(void) + proto_unregister(&qrtr_proto); + return rc; + } +- rtnl_register(PF_QIPCRTR, RTM_NEWADDR, qrtr_addr_doit, NULL, NULL); ++ ++ qrtr_ns_init(); + + return 0; + } +@@ -1819,7 +1795,8 @@ postcore_initcall(qrtr_proto_init); + + static void __exit qrtr_proto_fini(void) + { +- rtnl_unregister(PF_QIPCRTR, RTM_NEWADDR); ++ cancel_delayed_work_sync(&qrtr_ns_work); ++ qrtr_ns_remove(); + sock_unregister(qrtr_family.family); + proto_unregister(&qrtr_proto); + } +Index: linux-4.4.60/net/qrtr/qrtr.h +=================================================================== +--- linux-4.4.60.orig/net/qrtr/qrtr.h ++++ linux-4.4.60/net/qrtr/qrtr.h +@@ -33,4 +33,9 @@ void qrtr_endpoint_unregister(struct qrt + int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len); + + int qrtr_peek_pkt_size(const void *data); ++ ++void qrtr_ns_init(void); ++ ++void qrtr_ns_remove(void); ++ + #endif +Index: linux-4.4.60/net/qrtr/Makefile +=================================================================== +--- linux-4.4.60.orig/net/qrtr/Makefile ++++ linux-4.4.60/net/qrtr/Makefile +@@ -1,4 +1,4 @@ +-obj-$(CONFIG_QRTR) := qrtr.o ++obj-$(CONFIG_QRTR) := qrtr.o ns.o + + obj-$(CONFIG_QRTR_SMD) += qrtr-smd.o + qrtr-smd-y := smd.o +Index: linux-4.4.60/include/trace/events/qrtr.h +=================================================================== +--- /dev/null ++++ linux-4.4.60/include/trace/events/qrtr.h +@@ -0,0 +1,115 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM qrtr ++ ++#if !defined(_TRACE_QRTR_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_QRTR_H ++ ++#include ++#include ++ ++TRACE_EVENT(qrtr_ns_service_announce_new, ++ ++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), ++ ++ TP_ARGS(service, instance, node, port), ++ ++ TP_STRUCT__entry( ++ __field(__le32, service) ++ __field(__le32, instance) ++ __field(__le32, node) ++ __field(__le32, port) ++ ), ++ ++ TP_fast_assign( ++ __entry->service = service; ++ __entry->instance = instance; ++ __entry->node = node; ++ __entry->port = port; ++ ), ++ ++ TP_printk("advertising new server [%d:%x]@[%d:%d]", ++ __entry->service, __entry->instance, __entry->node, ++ __entry->port ++ ) ++); ++ ++TRACE_EVENT(qrtr_ns_service_announce_del, ++ ++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), ++ ++ TP_ARGS(service, instance, node, port), ++ ++ TP_STRUCT__entry( ++ __field(__le32, service) ++ __field(__le32, instance) ++ __field(__le32, node) ++ __field(__le32, port) ++ ), ++ ++ TP_fast_assign( ++ __entry->service = service; ++ __entry->instance = instance; ++ __entry->node = node; ++ __entry->port = port; ++ ), ++ ++ TP_printk("advertising removal of server [%d:%x]@[%d:%d]", ++ __entry->service, __entry->instance, __entry->node, ++ __entry->port ++ ) ++); ++ ++TRACE_EVENT(qrtr_ns_server_add, ++ ++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), ++ ++ TP_ARGS(service, instance, node, port), ++ ++ TP_STRUCT__entry( ++ __field(__le32, service) ++ __field(__le32, instance) ++ __field(__le32, node) ++ __field(__le32, port) ++ ), ++ ++ TP_fast_assign( ++ __entry->service = service; ++ __entry->instance = instance; ++ __entry->node = node; ++ __entry->port = port; ++ ), ++ ++ TP_printk("add server [%d:%x]@[%d:%d]", ++ __entry->service, __entry->instance, __entry->node, ++ __entry->port ++ ) ++); ++ ++TRACE_EVENT(qrtr_ns_message, ++ ++ TP_PROTO(const char * const ctrl_pkt_str, __u32 sq_node, __u32 sq_port), ++ ++ TP_ARGS(ctrl_pkt_str, sq_node, sq_port), ++ ++ TP_STRUCT__entry( ++ __string(ctrl_pkt_str, ctrl_pkt_str) ++ __field(__u32, sq_node) ++ __field(__u32, sq_port) ++ ), ++ ++ TP_fast_assign( ++ __assign_str(ctrl_pkt_str, ctrl_pkt_str); ++ __entry->sq_node = sq_node; ++ __entry->sq_port = sq_port; ++ ), ++ ++ TP_printk("%s from %d:%d", ++ __get_str(ctrl_pkt_str), __entry->sq_node, __entry->sq_port ++ ) ++); ++ ++#endif /* _TRACE_QRTR_H */ ++ ++/* This part must be outside protection */ ++#include diff --git a/feeds/ipq807x/ipq807x/patches/101-squashfs.patch b/feeds/ipq807x/ipq807x/patches/101-squashfs.patch new file mode 100644 index 000000000..5744a2a5b --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/101-squashfs.patch @@ -0,0 +1,16 @@ +Index: linux-4.4.60/fs/squashfs/xz_wrapper.c +=================================================================== +--- linux-4.4.60.orig/fs/squashfs/xz_wrapper.c ++++ linux-4.4.60/fs/squashfs/xz_wrapper.c +@@ -40,10 +40,8 @@ struct squashfs_xz { + }; + + struct disk_comp_opts { +- __le32 flags; +- __le16 bit_opts; +- __le16 fb; + __le32 dictionary_size; ++ __le32 flags; + }; + + struct comp_opts { diff --git a/feeds/ipq807x/ipq807x/patches/102-aq-phy.patch b/feeds/ipq807x/ipq807x/patches/102-aq-phy.patch new file mode 100644 index 000000000..6c6d9b88e --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/102-aq-phy.patch @@ -0,0 +1,90 @@ +Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/drivers/net/phy/aquantia.c +=================================================================== +--- linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1.orig/drivers/net/phy/aquantia.c ++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/drivers/net/phy/aquantia.c +@@ -32,6 +32,7 @@ + #define PHY_ID_AQR112 0x03a1b660 + #define PHY_ID_AQR113C 0x31c31C10 + #define PHY_ID_AQR112C 0x03a1b792 ++#define PHY_ID_AQR114C 0x31c31C22 + + #define AQ_PHY_MAX_VALID_MMD_REG 0xff01 + #define AQ_PHY_MAX_INVALID_MMD_REG 0xffff +@@ -756,6 +757,25 @@ static struct phy_driver aquantia_driver + .update_link = aquantia_update_link, + .driver = { .owner = THIS_MODULE,}, + }, ++{ ++ .phy_id = PHY_ID_AQR114C, ++ .phy_id_mask = 0xfffffff0, ++ .name = "Aquantia AQR114C", ++ .features = PHY_AQUANTIA_FEATURES, ++ .flags = PHY_HAS_INTERRUPT, ++ .probe = aquantia_phy_probe, ++ .soft_reset = aquantia_soft_reset, ++ .config_init = aquantia_config_init, ++ .aneg_done = aquantia_aneg_done, ++ .config_aneg = aquantia_config_aneg, ++ .config_intr = aquantia_config_intr, ++ .ack_interrupt = aquantia_ack_interrupt, ++ .read_status = aquantia_read_status, ++ .suspend = aquantia_suspend, ++ .resume = aquantia_resume, ++ .update_link = aquantia_update_link, ++ .driver = { .owner = THIS_MODULE,}, ++}, + }; + + module_phy_driver(aquantia_driver); +@@ -773,6 +793,7 @@ static struct mdio_device_id __maybe_unu + { PHY_ID_AQR112, 0xfffffff0 }, + { PHY_ID_AQR113C, 0xfffffff0 }, + { PHY_ID_AQR112C, 0xfffffff0 }, ++ { PHY_ID_AQR114C, 0xfffffff0 }, + { } + }; + +Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/net/phy/aquantia.c +=================================================================== +--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/net/phy/aquantia.c ++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/net/phy/aquantia.c +@@ -29,6 +29,7 @@ + #define PHY_ID_AQR109 0x03a1b502 + #define PHY_ID_AQR111 0x03a1b610 + #define PHY_ID_AQR111B0 0x03a1b612 ++#define PHY_ID_AQR111C 0x03a1b7e2 + #define PHY_ID_AQR112 0x03a1b660 + #define PHY_ID_AQR113C 0x31c31C10 + #define PHY_ID_AQR112C 0x03a1b792 +@@ -701,6 +702,23 @@ static struct phy_driver aquantia_driver + .driver = { .owner = THIS_MODULE,}, + }, + { ++ .phy_id = PHY_ID_AQR111C, ++ .phy_id_mask = 0xfffffff0, ++ .name = "Aquantia AQR111C", ++ .features = PHY_AQUANTIA_FEATURES, ++ .flags = PHY_HAS_INTERRUPT, ++ .probe = aquantia_phy_probe, ++ .soft_reset = aquantia_soft_reset, ++ .config_init = aquantia_config_init, ++ .aneg_done = aquantia_aneg_done, ++ .config_aneg = aquantia_config_aneg, ++ .config_intr = aquantia_config_intr, ++ .ack_interrupt = aquantia_ack_interrupt, ++ .read_status = aquantia_read_status, ++ .update_link = aquantia_update_link, ++ .driver = { .owner = THIS_MODULE,}, ++}, ++{ + .phy_id = PHY_ID_AQR112, + .phy_id_mask = 0xfffffff0, + .name = "Aquantia AQR112", +@@ -790,6 +808,7 @@ static struct mdio_device_id __maybe_unu + { PHY_ID_AQR109, 0xfffffff0 }, + { PHY_ID_AQR111, 0xfffffff0 }, + { PHY_ID_AQR111B0, 0xfffffff0 }, ++ { PHY_ID_AQR111C, 0xfffffff0 }, + { PHY_ID_AQR112, 0xfffffff0 }, + { PHY_ID_AQR113C, 0xfffffff0 }, + { PHY_ID_AQR112C, 0xfffffff0 }, diff --git a/feeds/ipq807x/ipq807x/patches/103-fix-dtc-gcc10-build.patch b/feeds/ipq807x/ipq807x/patches/103-fix-dtc-gcc10-build.patch new file mode 100644 index 000000000..f91601ec5 --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/103-fix-dtc-gcc10-build.patch @@ -0,0 +1,11 @@ +--- a/scripts/dtc/dtc-lexer.lex.c_shipped ++++ b/scripts/dtc/dtc-lexer.lex.c_shipped +@@ -637,7 +637,7 @@ char *yytext; + #include "srcpos.h" + #include "dtc-parser.tab.h" + +-YYLTYPE yylloc; ++extern YYLTYPE yylloc; + extern bool treesource_error; + + /* CAUTION: this will stop working if we ever use yyless() or yyunput() */ diff --git a/feeds/ipq807x/ipq807x/patches/104-log-spam.patch b/feeds/ipq807x/ipq807x/patches/104-log-spam.patch new file mode 100644 index 000000000..94b2a3ffd --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/104-log-spam.patch @@ -0,0 +1,37 @@ +Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c +=================================================================== +--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/clk/qcom/clk-branch.c ++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c +@@ -75,7 +75,7 @@ static int clk_branch_wait(const struct + bool (check_halt)(const struct clk_branch *, bool)) + { + bool voted = br->halt_check & BRANCH_VOTED; +- const char *name = clk_hw_get_name(&br->clkr.hw); ++ //const char *name = clk_hw_get_name(&br->clkr.hw); + + /* Skip checking halt bit if the clock is in hardware gated mode */ + if (clk_branch_in_hwcg_mode(br)) +@@ -93,8 +93,8 @@ static int clk_branch_wait(const struct + return 0; + udelay(1); + } +- WARN(1, "%s status stuck at 'o%s'", name, +- enabling ? "ff" : "n"); ++/* WARN(1, "%s status stuck at 'o%s'", name, ++ enabling ? "ff" : "n");*/ + return -EBUSY; + } + return 0; +Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/usb/phy/phy-msm-qusb.c +=================================================================== +--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/usb/phy/phy-msm-qusb.c ++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/usb/phy/phy-msm-qusb.c +@@ -491,7 +491,7 @@ static int qusb_phy_init(struct usb_phy + dev_err(phy->dev, "QUSB PHY PLL LOCK fails:%x\n", + readb_relaxed(qphy->base + + QUSB2PHY_PLL_STATUS)); +- WARN_ON(1); ++ //WARN_ON(1); + } + + /* Set OTG VBUS Valid from HSPHY to controller */ diff --git a/feeds/ipq807x/ipq807x/patches/105-add-esmt-nand.patch b/feeds/ipq807x/ipq807x/patches/105-add-esmt-nand.patch new file mode 100644 index 000000000..d47a4d0d1 --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/105-add-esmt-nand.patch @@ -0,0 +1,37 @@ +Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/mtd/nand/nand_ids.c +=================================================================== +--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/mtd/nand/nand_ids.c ++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/mtd/nand/nand_ids.c +@@ -62,6 +62,12 @@ struct nand_flash_dev nand_flash_ids[] = + {"TH58NYG3S0H 8G 1.8V 8-bit", + { .id = {0x98, 0xa3, 0x91, 0x26} }, + SZ_4K, SZ_1K, SZ_256K, 0, 4, 256, NAND_ECC_INFO(8, SZ_512) }, ++ ++ {"F59D2G81KA 2G 1.8V 8-bit", ++ { .id = {0xc8, 0x5a, 0x90, 0x04} }, ++ SZ_2K, SZ_256, SZ_128K, 0, 4, 128, NAND_ECC_INFO(8, SZ_512) }, ++ ++ + LEGACY_ID_NAND("NAND 4MiB 5V 8-bit", 0x6B, 4, SZ_8K, SP_OPTIONS), + LEGACY_ID_NAND("NAND 4MiB 3,3V 8-bit", 0xE3, 4, SZ_8K, SP_OPTIONS), + LEGACY_ID_NAND("NAND 4MiB 3,3V 8-bit", 0xE5, 4, SZ_8K, SP_OPTIONS), +@@ -190,6 +196,7 @@ struct nand_manufacturers nand_manuf_ids + {NAND_MFR_SANDISK, "SanDisk"}, + {NAND_MFR_INTEL, "Intel"}, + {NAND_MFR_ATO, "ATO"}, ++ {NAND_MFR_ESMT, "ESMT"}, + {NAND_MFR_GIGA, "GigaDevice"}, + {NAND_MFR_ATO, "ATO"}, + {NAND_MFR_WINBOND, "Winbond"}, +Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/include/linux/mtd/nand.h +=================================================================== +--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/include/linux/mtd/nand.h ++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/include/linux/mtd/nand.h +@@ -778,6 +778,7 @@ static inline struct mtd_info *nand_to_m + #define NAND_MFR_ATO 0x9b + #define NAND_MFR_WINBOND 0xef + #define NAND_MFR_FIDELIX 0xe5 ++#define NAND_MFR_ESMT 0xc8 + + /* The maximum expected count of bytes in the NAND ID sequence */ + #define NAND_MAX_ID_LEN 8 diff --git a/feeds/ipq807x/ipq807x/patches/106-pstore.patch b/feeds/ipq807x/ipq807x/patches/106-pstore.patch new file mode 100644 index 000000000..dc3960306 --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/106-pstore.patch @@ -0,0 +1,147 @@ +Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi +=================================================================== +--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi ++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi +@@ -92,6 +92,12 @@ + reg = <0x0 0x40000000 0x0 0x00800000>; + }; + ++ ramoops@4A0f0000 { ++ compatible = "ramoops"; ++ reg = <0 0x4A0f0000 0 0x10000>; ++ record-size = <0x1000>; ++ }; ++ + uboot@4A100000 { + no-map; + reg = <0x0 0x4A100000 0x0 0x00400000>; +@@ -211,6 +217,12 @@ + reg = <0x0 0x40000000 0x0 0x01000000>; + }; + ++ ramoops@4A0f0000 { ++ compatible = "ramoops"; ++ reg = <0 0x4A0f0000 0 0x10000>; ++ record-size = <0x1000>; ++ }; ++ + uboot@4A100000 { + no-map; + reg = <0x0 0x4A100000 0x0 0x00400000>; +@@ -330,6 +342,12 @@ + reg = <0x0 0x40000000 0x0 0x01000000>; + }; + ++ ramoops@4A0f0000 { ++ compatible = "ramoops"; ++ reg = <0 0x4A0f0000 0 0x10000>; ++ record-size = <0x1000>; ++ }; ++ + uboot@4A100000 { + no-map; + reg = <0x0 0x4A100000 0x0 0x00400000>; +Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/fs/pstore/ram.c +=================================================================== +--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/fs/pstore/ram.c ++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/fs/pstore/ram.c +@@ -466,15 +466,46 @@ static int ramoops_init_prz(struct devic + return 0; + } + ++static int ramoops_parse_dt(struct platform_device *pdev, ++ struct ramoops_platform_data *pdata) ++{ ++ struct resource *res; ++ ++ dev_dbg(&pdev->dev, "using Device Tree\n"); ++ ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ if (!res) { ++ dev_err(&pdev->dev, ++ "failed to locate DT /reserved-memory resource\n"); ++ return -EINVAL; ++ } ++ ++ pdata->mem_size = resource_size(res); ++ pdata->mem_address = res->start; ++ pdata->dump_oops = true; ++ pdata->record_size = 0x1000; ++ return 0; ++} ++ + static int ramoops_probe(struct platform_device *pdev) + { + struct device *dev = &pdev->dev; + struct ramoops_platform_data *pdata = pdev->dev.platform_data; ++ struct ramoops_platform_data pdata_local; + struct ramoops_context *cxt = &oops_cxt; + size_t dump_mem_sz; + phys_addr_t paddr; + int err = -EINVAL; + ++ if (dev_of_node(dev) && !pdata) { ++ pdata = &pdata_local; ++ memset(pdata, 0, sizeof(*pdata)); ++ ++ err = ramoops_parse_dt(pdev, pdata); ++ if (err < 0) ++ goto fail_out; ++ } ++ + /* Only a single ramoops area allowed at a time, so fail extra + * probes. + */ +@@ -603,11 +634,17 @@ static int ramoops_remove(struct platfor + return 0; + } + ++static const struct of_device_id dt_match[] = { ++ { .compatible = "ramoops" }, ++ {} ++}; ++ + static struct platform_driver ramoops_driver = { + .probe = ramoops_probe, + .remove = ramoops_remove, + .driver = { + .name = "ramoops", ++ .of_match_table = dt_match, + }, + }; + +Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/drivers/of/platform.c +=================================================================== +--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/drivers/of/platform.c ++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/drivers/of/platform.c +@@ -53,6 +53,30 @@ struct platform_device *of_find_device_b + } + EXPORT_SYMBOL(of_find_device_by_node); + ++static const struct of_device_id reserved_mem_matches[] = { ++ { .compatible = "ramoops" }, ++ {} ++}; ++ ++static int __init of_platform_default_populate_init(void) ++{ ++ struct device_node *node; ++ ++ if (!of_have_populated_dt()) ++ return -ENODEV; ++ ++ /* ++ * Handle certain compatibles explicitly, since we don't want to create ++ * platform_devices for every node in /reserved-memory with a ++ * "compatible", ++ */ ++ for_each_matching_node(node, reserved_mem_matches) ++ of_platform_device_create(node, NULL, NULL); ++ ++ return 0; ++} ++arch_initcall_sync(of_platform_default_populate_init); ++ + #ifdef CONFIG_OF_ADDRESS + /* + * The following routines scan a subtree and registers a device for diff --git a/feeds/ipq807x/ipq807x/patches/200-bpf_backport.patch b/feeds/ipq807x/ipq807x/patches/200-bpf_backport.patch new file mode 100644 index 000000000..4357369c2 --- /dev/null +++ b/feeds/ipq807x/ipq807x/patches/200-bpf_backport.patch @@ -0,0 +1,44780 @@ +--- a/arch/arm/Kconfig ++++ b/arch/arm/Kconfig +@@ -38,7 +38,7 @@ config ARM + select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 + select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT) + select HAVE_ARCH_TRACEHOOK +- select HAVE_BPF_JIT ++ select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 + select HAVE_CC_STACKPROTECTOR + select HAVE_CONTEXT_TRACKING + select HAVE_C_RECORDMCOUNT +--- a/arch/arm/net/bpf_jit_32.c ++++ b/arch/arm/net/bpf_jit_32.c +@@ -1,13 +1,12 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* +- * Just-In-Time compiler for BPF filters on 32bit ARM ++ * Just-In-Time compiler for eBPF filters on 32bit ARM + * ++ * Copyright (c) 2017 Shubham Bansal + * Copyright (c) 2011 Mircea Gherzan +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. + */ + ++#include + #include + #include + #include +@@ -20,51 +19,182 @@ + #include + #include + #include ++#include + + #include "bpf_jit_32.h" + + /* +- * ABI: ++ * eBPF prog stack layout: ++ * ++ * high ++ * original ARM_SP => +-----+ ++ * | | callee saved registers ++ * +-----+ <= (BPF_FP + SCRATCH_SIZE) ++ * | ... | eBPF JIT scratch space ++ * eBPF fp register => +-----+ ++ * (BPF_FP) | ... | eBPF prog stack ++ * +-----+ ++ * |RSVD | JIT scratchpad ++ * current ARM_SP => +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE) ++ * | | ++ * | ... | Function call stack ++ * | | ++ * +-----+ ++ * low ++ * ++ * The callee saved registers depends on whether frame pointers are enabled. ++ * With frame pointers (to be compliant with the ABI): ++ * ++ * high ++ * original ARM_SP => +--------------+ \ ++ * | pc | | ++ * current ARM_FP => +--------------+ } callee saved registers ++ * |r4-r9,fp,ip,lr| | ++ * +--------------+ / ++ * low + * +- * r0 scratch register +- * r4 BPF register A +- * r5 BPF register X +- * r6 pointer to the skb +- * r7 skb->data +- * r8 skb_headlen(skb) ++ * Without frame pointers: ++ * ++ * high ++ * original ARM_SP => +--------------+ ++ * | r4-r9,fp,lr | callee saved registers ++ * current ARM_FP => +--------------+ ++ * low ++ * ++ * When popping registers off the stack at the end of a BPF function, we ++ * reference them via the current ARM_FP register. + */ ++#define CALLEE_MASK (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \ ++ 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R9 | \ ++ 1 << ARM_FP) ++#define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR) ++#define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC) ++ ++enum { ++ /* Stack layout - these are offsets from (top of stack - 4) */ ++ BPF_R2_HI, ++ BPF_R2_LO, ++ BPF_R3_HI, ++ BPF_R3_LO, ++ BPF_R4_HI, ++ BPF_R4_LO, ++ BPF_R5_HI, ++ BPF_R5_LO, ++ BPF_R7_HI, ++ BPF_R7_LO, ++ BPF_R8_HI, ++ BPF_R8_LO, ++ BPF_R9_HI, ++ BPF_R9_LO, ++ BPF_FP_HI, ++ BPF_FP_LO, ++ BPF_TC_HI, ++ BPF_TC_LO, ++ BPF_AX_HI, ++ BPF_AX_LO, ++ /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4, ++ * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, ++ * BPF_REG_FP and Tail call counts. ++ */ ++ BPF_JIT_SCRATCH_REGS, ++}; ++ ++/* ++ * Negative "register" values indicate the register is stored on the stack ++ * and are the offset from the top of the eBPF JIT scratch space. ++ */ ++#define STACK_OFFSET(k) (-4 - (k) * 4) ++#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4) ++ ++#ifdef CONFIG_FRAME_POINTER ++#define EBPF_SCRATCH_TO_ARM_FP(x) ((x) - 4 * hweight16(CALLEE_PUSH_MASK) - 4) ++#else ++#define EBPF_SCRATCH_TO_ARM_FP(x) (x) ++#endif + +-#define r_scratch ARM_R0 +-/* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */ +-#define r_off ARM_R1 +-#define r_A ARM_R4 +-#define r_X ARM_R5 +-#define r_skb ARM_R6 +-#define r_skb_data ARM_R7 +-#define r_skb_hl ARM_R8 +- +-#define SCRATCH_SP_OFFSET 0 +-#define SCRATCH_OFF(k) (SCRATCH_SP_OFFSET + 4 * (k)) +- +-#define SEEN_MEM ((1 << BPF_MEMWORDS) - 1) +-#define SEEN_MEM_WORD(k) (1 << (k)) +-#define SEEN_X (1 << BPF_MEMWORDS) +-#define SEEN_CALL (1 << (BPF_MEMWORDS + 1)) +-#define SEEN_SKB (1 << (BPF_MEMWORDS + 2)) +-#define SEEN_DATA (1 << (BPF_MEMWORDS + 3)) ++#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ ++#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */ ++#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */ + +-#define FLAG_NEED_X_RESET (1 << 0) +-#define FLAG_IMM_OVERFLOW (1 << 1) ++#define FLAG_IMM_OVERFLOW (1 << 0) ++ ++/* ++ * Map eBPF registers to ARM 32bit registers or stack scratch space. ++ * ++ * 1. First argument is passed using the arm 32bit registers and rest of the ++ * arguments are passed on stack scratch space. ++ * 2. First callee-saved argument is mapped to arm 32 bit registers and rest ++ * arguments are mapped to scratch space on stack. ++ * 3. We need two 64 bit temp registers to do complex operations on eBPF ++ * registers. ++ * ++ * As the eBPF registers are all 64 bit registers and arm has only 32 bit ++ * registers, we have to map each eBPF registers with two arm 32 bit regs or ++ * scratch memory space and we have to build eBPF 64 bit register from those. ++ * ++ */ ++static const s8 bpf2a32[][2] = { ++ /* return value from in-kernel function, and exit value from eBPF */ ++ [BPF_REG_0] = {ARM_R1, ARM_R0}, ++ /* arguments from eBPF program to in-kernel function */ ++ [BPF_REG_1] = {ARM_R3, ARM_R2}, ++ /* Stored on stack scratch space */ ++ [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)}, ++ [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)}, ++ [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)}, ++ [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)}, ++ /* callee saved registers that in-kernel function will preserve */ ++ [BPF_REG_6] = {ARM_R5, ARM_R4}, ++ /* Stored on stack scratch space */ ++ [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)}, ++ [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, ++ [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, ++ /* Read only Frame Pointer to access Stack */ ++ [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)}, ++ /* Temporary Register for internal BPF JIT, can be used ++ * for constant blindings and others. ++ */ ++ [TMP_REG_1] = {ARM_R7, ARM_R6}, ++ [TMP_REG_2] = {ARM_R9, ARM_R8}, ++ /* Tail call count. Stored on stack scratch space. */ ++ [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)}, ++ /* temporary register for blinding constants. ++ * Stored on stack scratch space. ++ */ ++ [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, ++}; ++ ++#define dst_lo dst[1] ++#define dst_hi dst[0] ++#define src_lo src[1] ++#define src_hi src[0] ++ ++/* ++ * JIT Context: ++ * ++ * prog : bpf_prog ++ * idx : index of current last JITed instruction. ++ * prologue_bytes : bytes used in prologue. ++ * epilogue_offset : offset of epilogue starting. ++ * offsets : array of eBPF instruction offsets in ++ * JITed code. ++ * target : final JITed code. ++ * epilogue_bytes : no of bytes used in epilogue. ++ * imm_count : no of immediate counts used for global ++ * variables. ++ * imms : array of global variable addresses. ++ */ + + struct jit_ctx { +- const struct bpf_prog *skf; +- unsigned idx; +- unsigned prologue_bytes; +- int ret0_fp_idx; +- u32 seen; ++ const struct bpf_prog *prog; ++ unsigned int idx; ++ unsigned int prologue_bytes; ++ unsigned int epilogue_offset; ++ unsigned int cpu_architecture; + u32 flags; + u32 *offsets; + u32 *target; ++ u32 stack_size; + #if __LINUX_ARM_ARCH__ < 7 + u16 epilogue_bytes; + u16 imm_count; +@@ -72,68 +202,16 @@ struct jit_ctx { + #endif + }; + +-int bpf_jit_enable __read_mostly; +- +-static inline int call_neg_helper(struct sk_buff *skb, int offset, void *ret, +- unsigned int size) +-{ +- void *ptr = bpf_internal_load_pointer_neg_helper(skb, offset, size); +- +- if (!ptr) +- return -EFAULT; +- memcpy(ret, ptr, size); +- return 0; +-} +- +-static u64 jit_get_skb_b(struct sk_buff *skb, int offset) +-{ +- u8 ret; +- int err; +- +- if (offset < 0) +- err = call_neg_helper(skb, offset, &ret, 1); +- else +- err = skb_copy_bits(skb, offset, &ret, 1); +- +- return (u64)err << 32 | ret; +-} +- +-static u64 jit_get_skb_h(struct sk_buff *skb, int offset) +-{ +- u16 ret; +- int err; +- +- if (offset < 0) +- err = call_neg_helper(skb, offset, &ret, 2); +- else +- err = skb_copy_bits(skb, offset, &ret, 2); +- +- return (u64)err << 32 | ntohs(ret); +-} +- +-static u64 jit_get_skb_w(struct sk_buff *skb, int offset) +-{ +- u32 ret; +- int err; +- +- if (offset < 0) +- err = call_neg_helper(skb, offset, &ret, 4); +- else +- err = skb_copy_bits(skb, offset, &ret, 4); +- +- return (u64)err << 32 | ntohl(ret); +-} +- + /* + * Wrappers which handle both OABI and EABI and assures Thumb2 interworking + * (where the assembly routines like __aeabi_uidiv could cause problems). + */ +-static u32 jit_udiv(u32 dividend, u32 divisor) ++static u32 jit_udiv32(u32 dividend, u32 divisor) + { + return dividend / divisor; + } + +-static u32 jit_mod(u32 dividend, u32 divisor) ++static u32 jit_mod32(u32 dividend, u32 divisor) + { + return dividend % divisor; + } +@@ -157,36 +235,100 @@ static inline void emit(u32 inst, struct + _emit(ARM_COND_AL, inst, ctx); + } + +-static u16 saved_regs(struct jit_ctx *ctx) ++/* ++ * This is rather horrid, but necessary to convert an integer constant ++ * to an immediate operand for the opcodes, and be able to detect at ++ * build time whether the constant can't be converted (iow, usable in ++ * BUILD_BUG_ON()). ++ */ ++#define imm12val(v, s) (rol32(v, (s)) | (s) << 7) ++#define const_imm8m(x) \ ++ ({ int r; \ ++ u32 v = (x); \ ++ if (!(v & ~0x000000ff)) \ ++ r = imm12val(v, 0); \ ++ else if (!(v & ~0xc000003f)) \ ++ r = imm12val(v, 2); \ ++ else if (!(v & ~0xf000000f)) \ ++ r = imm12val(v, 4); \ ++ else if (!(v & ~0xfc000003)) \ ++ r = imm12val(v, 6); \ ++ else if (!(v & ~0xff000000)) \ ++ r = imm12val(v, 8); \ ++ else if (!(v & ~0x3fc00000)) \ ++ r = imm12val(v, 10); \ ++ else if (!(v & ~0x0ff00000)) \ ++ r = imm12val(v, 12); \ ++ else if (!(v & ~0x03fc0000)) \ ++ r = imm12val(v, 14); \ ++ else if (!(v & ~0x00ff0000)) \ ++ r = imm12val(v, 16); \ ++ else if (!(v & ~0x003fc000)) \ ++ r = imm12val(v, 18); \ ++ else if (!(v & ~0x000ff000)) \ ++ r = imm12val(v, 20); \ ++ else if (!(v & ~0x0003fc00)) \ ++ r = imm12val(v, 22); \ ++ else if (!(v & ~0x0000ff00)) \ ++ r = imm12val(v, 24); \ ++ else if (!(v & ~0x00003fc0)) \ ++ r = imm12val(v, 26); \ ++ else if (!(v & ~0x00000ff0)) \ ++ r = imm12val(v, 28); \ ++ else if (!(v & ~0x000003fc)) \ ++ r = imm12val(v, 30); \ ++ else \ ++ r = -1; \ ++ r; }) ++ ++/* ++ * Checks if immediate value can be converted to imm12(12 bits) value. ++ */ ++static int imm8m(u32 x) + { +- u16 ret = 0; ++ u32 rot; + +- if ((ctx->skf->len > 1) || +- (ctx->skf->insns[0].code == (BPF_RET | BPF_A))) +- ret |= 1 << r_A; ++ for (rot = 0; rot < 16; rot++) ++ if ((x & ~ror32(0xff, 2 * rot)) == 0) ++ return rol32(x, 2 * rot) | (rot << 8); ++ return -1; ++} + +-#ifdef CONFIG_FRAME_POINTER +- ret |= (1 << ARM_FP) | (1 << ARM_IP) | (1 << ARM_LR) | (1 << ARM_PC); +-#else +- if (ctx->seen & SEEN_CALL) +- ret |= 1 << ARM_LR; +-#endif +- if (ctx->seen & (SEEN_DATA | SEEN_SKB)) +- ret |= 1 << r_skb; +- if (ctx->seen & SEEN_DATA) +- ret |= (1 << r_skb_data) | (1 << r_skb_hl); +- if (ctx->seen & SEEN_X) +- ret |= 1 << r_X; ++#define imm8m(x) (__builtin_constant_p(x) ? const_imm8m(x) : imm8m(x)) + +- return ret; ++static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12) ++{ ++ op |= rt << 12 | rn << 16; ++ if (imm12 >= 0) ++ op |= ARM_INST_LDST__U; ++ else ++ imm12 = -imm12; ++ return op | (imm12 & ARM_INST_LDST__IMM12); + } + +-static inline int mem_words_used(struct jit_ctx *ctx) ++static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8) + { +- /* yes, we do waste some stack space IF there are "holes" in the set" */ +- return fls(ctx->seen & SEEN_MEM); ++ op |= rt << 12 | rn << 16; ++ if (imm8 >= 0) ++ op |= ARM_INST_LDST__U; ++ else ++ imm8 = -imm8; ++ return op | (imm8 & 0xf0) << 4 | (imm8 & 0x0f); + } + ++#define ARM_LDR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off) ++#define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off) ++#define ARM_LDRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off) ++#define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off) ++ ++#define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off) ++#define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off) ++#define ARM_STRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRD_I, rt, rn, off) ++#define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off) ++ ++/* ++ * Initializes the JIT space with undefined instructions. ++ */ + static void jit_fill_hole(void *area, unsigned int size) + { + u32 *ptr; +@@ -195,88 +337,23 @@ static void jit_fill_hole(void *area, un + *ptr++ = __opcode_to_mem_arm(ARM_INST_UDF); + } + +-static void build_prologue(struct jit_ctx *ctx) +-{ +- u16 reg_set = saved_regs(ctx); +- u16 off; +- +-#ifdef CONFIG_FRAME_POINTER +- emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); +- emit(ARM_PUSH(reg_set), ctx); +- emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); ++#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) ++/* EABI requires the stack to be aligned to 64-bit boundaries */ ++#define STACK_ALIGNMENT 8 + #else +- if (reg_set) +- emit(ARM_PUSH(reg_set), ctx); ++/* Stack must be aligned to 32-bit boundaries */ ++#define STACK_ALIGNMENT 4 + #endif + +- if (ctx->seen & (SEEN_DATA | SEEN_SKB)) +- emit(ARM_MOV_R(r_skb, ARM_R0), ctx); +- +- if (ctx->seen & SEEN_DATA) { +- off = offsetof(struct sk_buff, data); +- emit(ARM_LDR_I(r_skb_data, r_skb, off), ctx); +- /* headlen = len - data_len */ +- off = offsetof(struct sk_buff, len); +- emit(ARM_LDR_I(r_skb_hl, r_skb, off), ctx); +- off = offsetof(struct sk_buff, data_len); +- emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); +- emit(ARM_SUB_R(r_skb_hl, r_skb_hl, r_scratch), ctx); +- } +- +- if (ctx->flags & FLAG_NEED_X_RESET) +- emit(ARM_MOV_I(r_X, 0), ctx); +- +- /* do not leak kernel data to userspace */ +- if (bpf_needs_clear_a(&ctx->skf->insns[0])) +- emit(ARM_MOV_I(r_A, 0), ctx); +- +- /* stack space for the BPF_MEM words */ +- if (ctx->seen & SEEN_MEM) +- emit(ARM_SUB_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); +-} +- +-static void build_epilogue(struct jit_ctx *ctx) +-{ +- u16 reg_set = saved_regs(ctx); +- +- if (ctx->seen & SEEN_MEM) +- emit(ARM_ADD_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); +- +- reg_set &= ~(1 << ARM_LR); +- +-#ifdef CONFIG_FRAME_POINTER +- /* the first instruction of the prologue was: mov ip, sp */ +- reg_set &= ~(1 << ARM_IP); +- reg_set |= (1 << ARM_SP); +- emit(ARM_LDM(ARM_SP, reg_set), ctx); +-#else +- if (reg_set) { +- if (ctx->seen & SEEN_CALL) +- reg_set |= 1 << ARM_PC; +- emit(ARM_POP(reg_set), ctx); +- } +- +- if (!(ctx->seen & SEEN_CALL)) +- emit(ARM_BX(ARM_LR), ctx); +-#endif +-} +- +-static int16_t imm8m(u32 x) +-{ +- u32 rot; +- +- for (rot = 0; rot < 16; rot++) +- if ((x & ~ror32(0xff, 2 * rot)) == 0) +- return rol32(x, 2 * rot) | (rot << 8); +- +- return -1; +-} ++/* total stack size used in JITed code */ ++#define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE) ++#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT) + + #if __LINUX_ARM_ARCH__ < 7 + + static u16 imm_offset(u32 k, struct jit_ctx *ctx) + { +- unsigned i = 0, offset; ++ unsigned int i = 0, offset; + u16 imm; + + /* on the "fake" run we just count them (duplicates included) */ +@@ -295,7 +372,7 @@ static u16 imm_offset(u32 k, struct jit_ + ctx->imms[i] = k; + + /* constants go just after the epilogue */ +- offset = ctx->offsets[ctx->skf->len]; ++ offset = ctx->offsets[ctx->prog->len - 1] * 4; + offset += ctx->prologue_bytes; + offset += ctx->epilogue_bytes; + offset += i * 4; +@@ -319,10 +396,22 @@ static u16 imm_offset(u32 k, struct jit_ + + #endif /* __LINUX_ARM_ARCH__ */ + ++static inline int bpf2a32_offset(int bpf_to, int bpf_from, ++ const struct jit_ctx *ctx) { ++ int to, from; ++ ++ if (ctx->target == NULL) ++ return 0; ++ to = ctx->offsets[bpf_to]; ++ from = ctx->offsets[bpf_from]; ++ ++ return to - from - 1; ++} ++ + /* + * Move an immediate that's not an imm8m to a core register. + */ +-static inline void emit_mov_i_no8m(int rd, u32 val, struct jit_ctx *ctx) ++static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx) + { + #if __LINUX_ARM_ARCH__ < 7 + emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); +@@ -333,7 +422,7 @@ static inline void emit_mov_i_no8m(int r + #endif + } + +-static inline void emit_mov_i(int rd, u32 val, struct jit_ctx *ctx) ++static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx) + { + int imm12 = imm8m(val); + +@@ -343,676 +432,1508 @@ static inline void emit_mov_i(int rd, u3 + emit_mov_i_no8m(rd, val, ctx); + } + +-#if __LINUX_ARM_ARCH__ < 6 ++static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx) ++{ ++ if (elf_hwcap & HWCAP_THUMB) ++ emit(ARM_BX(tgt_reg), ctx); ++ else ++ emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); ++} + +-static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) ++static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) + { +- _emit(cond, ARM_LDRB_I(ARM_R3, r_addr, 1), ctx); +- _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); +- _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 3), ctx); +- _emit(cond, ARM_LSL_I(ARM_R3, ARM_R3, 16), ctx); +- _emit(cond, ARM_LDRB_I(ARM_R0, r_addr, 2), ctx); +- _emit(cond, ARM_ORR_S(ARM_R3, ARM_R3, ARM_R1, SRTYPE_LSL, 24), ctx); +- _emit(cond, ARM_ORR_R(ARM_R3, ARM_R3, ARM_R2), ctx); +- _emit(cond, ARM_ORR_S(r_res, ARM_R3, ARM_R0, SRTYPE_LSL, 8), ctx); ++#if __LINUX_ARM_ARCH__ < 5 ++ emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); ++ emit_bx_r(tgt_reg, ctx); ++#else ++ emit(ARM_BLX_R(tgt_reg), ctx); ++#endif + } + +-static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) ++static inline int epilogue_offset(const struct jit_ctx *ctx) + { +- _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); +- _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 1), ctx); +- _emit(cond, ARM_ORR_S(r_res, ARM_R2, ARM_R1, SRTYPE_LSL, 8), ctx); ++ int to, from; ++ /* No need for 1st dummy run */ ++ if (ctx->target == NULL) ++ return 0; ++ to = ctx->epilogue_offset; ++ from = ctx->idx; ++ ++ return to - from - 2; + } + +-static inline void emit_swap16(u8 r_dst, u8 r_src, struct jit_ctx *ctx) ++static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) + { +- /* r_dst = (r_src << 8) | (r_src >> 8) */ +- emit(ARM_LSL_I(ARM_R1, r_src, 8), ctx); +- emit(ARM_ORR_S(r_dst, ARM_R1, r_src, SRTYPE_LSR, 8), ctx); ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ ++#if __LINUX_ARM_ARCH__ == 7 ++ if (elf_hwcap & HWCAP_IDIVA) { ++ if (op == BPF_DIV) ++ emit(ARM_UDIV(rd, rm, rn), ctx); ++ else { ++ emit(ARM_UDIV(ARM_IP, rm, rn), ctx); ++ emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); ++ } ++ return; ++ } ++#endif + + /* +- * we need to mask out the bits set in r_dst[23:16] due to +- * the first shift instruction. +- * +- * note that 0x8ff is the encoded immediate 0x00ff0000. ++ * For BPF_ALU | BPF_DIV | BPF_K instructions ++ * As ARM_R1 and ARM_R0 contains 1st argument of bpf ++ * function, we need to save it on caller side to save ++ * it from getting destroyed within callee. ++ * After the return from the callee, we restore ARM_R0 ++ * ARM_R1. + */ +- emit(ARM_BIC_I(r_dst, r_dst, 0x8ff), ctx); ++ if (rn != ARM_R1) { ++ emit(ARM_MOV_R(tmp[0], ARM_R1), ctx); ++ emit(ARM_MOV_R(ARM_R1, rn), ctx); ++ } ++ if (rm != ARM_R0) { ++ emit(ARM_MOV_R(tmp[1], ARM_R0), ctx); ++ emit(ARM_MOV_R(ARM_R0, rm), ctx); ++ } ++ ++ /* Call appropriate function */ ++ emit_mov_i(ARM_IP, op == BPF_DIV ? ++ (u32)jit_udiv32 : (u32)jit_mod32, ctx); ++ emit_blx_r(ARM_IP, ctx); ++ ++ /* Save return value */ ++ if (rd != ARM_R0) ++ emit(ARM_MOV_R(rd, ARM_R0), ctx); ++ ++ /* Restore ARM_R0 and ARM_R1 */ ++ if (rn != ARM_R1) ++ emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); ++ if (rm != ARM_R0) ++ emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); + } + +-#else /* ARMv6+ */ ++/* Is the translated BPF register on stack? */ ++static bool is_stacked(s8 reg) ++{ ++ return reg < 0; ++} + +-static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) ++/* If a BPF register is on the stack (stk is true), load it to the ++ * supplied temporary register and return the temporary register ++ * for subsequent operations, otherwise just use the CPU register. ++ */ ++static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) + { +- _emit(cond, ARM_LDR_I(r_res, r_addr, 0), ctx); +-#ifdef __LITTLE_ENDIAN +- _emit(cond, ARM_REV(r_res, r_res), ctx); +-#endif ++ if (is_stacked(reg)) { ++ emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); ++ reg = tmp; ++ } ++ return reg; + } + +-static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) ++static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, ++ struct jit_ctx *ctx) + { +- _emit(cond, ARM_LDRH_I(r_res, r_addr, 0), ctx); +-#ifdef __LITTLE_ENDIAN +- _emit(cond, ARM_REV16(r_res, r_res), ctx); +-#endif ++ if (is_stacked(reg[1])) { ++ if (__LINUX_ARM_ARCH__ >= 6 || ++ ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { ++ emit(ARM_LDRD_I(tmp[1], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); ++ } else { ++ emit(ARM_LDR_I(tmp[1], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); ++ emit(ARM_LDR_I(tmp[0], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); ++ } ++ reg = tmp; ++ } ++ return reg; + } + +-static inline void emit_swap16(u8 r_dst __maybe_unused, +- u8 r_src __maybe_unused, +- struct jit_ctx *ctx __maybe_unused) ++/* If a BPF register is on the stack (stk is true), save the register ++ * back to the stack. If the source register is not the same, then ++ * move it into the correct register. ++ */ ++static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) + { +-#ifdef __LITTLE_ENDIAN +- emit(ARM_REV16(r_dst, r_src), ctx); +-#endif ++ if (is_stacked(reg)) ++ emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); ++ else if (reg != src) ++ emit(ARM_MOV_R(reg, src), ctx); ++} ++ ++static void arm_bpf_put_reg64(const s8 *reg, const s8 *src, ++ struct jit_ctx *ctx) ++{ ++ if (is_stacked(reg[1])) { ++ if (__LINUX_ARM_ARCH__ >= 6 || ++ ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { ++ emit(ARM_STRD_I(src[1], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); ++ } else { ++ emit(ARM_STR_I(src[1], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); ++ emit(ARM_STR_I(src[0], ARM_FP, ++ EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); ++ } ++ } else { ++ if (reg[1] != src[1]) ++ emit(ARM_MOV_R(reg[1], src[1]), ctx); ++ if (reg[0] != src[0]) ++ emit(ARM_MOV_R(reg[0], src[0]), ctx); ++ } + } + +-#endif /* __LINUX_ARM_ARCH__ < 6 */ ++static inline void emit_a32_mov_i(const s8 dst, const u32 val, ++ struct jit_ctx *ctx) ++{ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; + ++ if (is_stacked(dst)) { ++ emit_mov_i(tmp[1], val, ctx); ++ arm_bpf_put_reg32(dst, tmp[1], ctx); ++ } else { ++ emit_mov_i(dst, val, ctx); ++ } ++} + +-/* Compute the immediate value for a PC-relative branch. */ +-static inline u32 b_imm(unsigned tgt, struct jit_ctx *ctx) ++static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx) + { +- u32 imm; ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *rd = is_stacked(dst_lo) ? tmp : dst; + +- if (ctx->target == NULL) +- return 0; +- /* +- * BPF allows only forward jumps and the offset of the target is +- * still the one computed during the first pass. ++ emit_mov_i(rd[1], (u32)val, ctx); ++ emit_mov_i(rd[0], val >> 32, ctx); ++ ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++/* Sign extended move */ ++static inline void emit_a32_mov_se_i64(const bool is64, const s8 dst[], ++ const u32 val, struct jit_ctx *ctx) { ++ u64 val64 = val; ++ ++ if (is64 && (val & (1<<31))) ++ val64 |= 0xffffffff00000000ULL; ++ emit_a32_mov_i64(dst, val64, ctx); ++} ++ ++static inline void emit_a32_add_r(const u8 dst, const u8 src, ++ const bool is64, const bool hi, ++ struct jit_ctx *ctx) { ++ /* 64 bit : ++ * adds dst_lo, dst_lo, src_lo ++ * adc dst_hi, dst_hi, src_hi ++ * 32 bit : ++ * add dst_lo, dst_lo, src_lo + */ +- imm = ctx->offsets[tgt] + ctx->prologue_bytes - (ctx->idx * 4 + 8); ++ if (!hi && is64) ++ emit(ARM_ADDS_R(dst, dst, src), ctx); ++ else if (hi && is64) ++ emit(ARM_ADC_R(dst, dst, src), ctx); ++ else ++ emit(ARM_ADD_R(dst, dst, src), ctx); ++} + +- return imm >> 2; ++static inline void emit_a32_sub_r(const u8 dst, const u8 src, ++ const bool is64, const bool hi, ++ struct jit_ctx *ctx) { ++ /* 64 bit : ++ * subs dst_lo, dst_lo, src_lo ++ * sbc dst_hi, dst_hi, src_hi ++ * 32 bit : ++ * sub dst_lo, dst_lo, src_lo ++ */ ++ if (!hi && is64) ++ emit(ARM_SUBS_R(dst, dst, src), ctx); ++ else if (hi && is64) ++ emit(ARM_SBC_R(dst, dst, src), ctx); ++ else ++ emit(ARM_SUB_R(dst, dst, src), ctx); ++} ++ ++static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, ++ const bool hi, const u8 op, struct jit_ctx *ctx){ ++ switch (BPF_OP(op)) { ++ /* dst = dst + src */ ++ case BPF_ADD: ++ emit_a32_add_r(dst, src, is64, hi, ctx); ++ break; ++ /* dst = dst - src */ ++ case BPF_SUB: ++ emit_a32_sub_r(dst, src, is64, hi, ctx); ++ break; ++ /* dst = dst | src */ ++ case BPF_OR: ++ emit(ARM_ORR_R(dst, dst, src), ctx); ++ break; ++ /* dst = dst & src */ ++ case BPF_AND: ++ emit(ARM_AND_R(dst, dst, src), ctx); ++ break; ++ /* dst = dst ^ src */ ++ case BPF_XOR: ++ emit(ARM_EOR_R(dst, dst, src), ctx); ++ break; ++ /* dst = dst * src */ ++ case BPF_MUL: ++ emit(ARM_MUL(dst, dst, src), ctx); ++ break; ++ /* dst = dst << src */ ++ case BPF_LSH: ++ emit(ARM_LSL_R(dst, dst, src), ctx); ++ break; ++ /* dst = dst >> src */ ++ case BPF_RSH: ++ emit(ARM_LSR_R(dst, dst, src), ctx); ++ break; ++ /* dst = dst >> src (signed)*/ ++ case BPF_ARSH: ++ emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx); ++ break; ++ } + } + +-#define OP_IMM3(op, r1, r2, imm_val, ctx) \ +- do { \ +- imm12 = imm8m(imm_val); \ +- if (imm12 < 0) { \ +- emit_mov_i_no8m(r_scratch, imm_val, ctx); \ +- emit(op ## _R((r1), (r2), r_scratch), ctx); \ +- } else { \ +- emit(op ## _I((r1), (r2), imm12), ctx); \ +- } \ +- } while (0) +- +-static inline void emit_err_ret(u8 cond, struct jit_ctx *ctx) +-{ +- if (ctx->ret0_fp_idx >= 0) { +- _emit(cond, ARM_B(b_imm(ctx->ret0_fp_idx, ctx)), ctx); +- /* NOP to keep the size constant between passes */ +- emit(ARM_MOV_R(ARM_R0, ARM_R0), ctx); ++/* ALU operation (32 bit) ++ * dst = dst (op) src ++ */ ++static inline void emit_a32_alu_r(const s8 dst, const s8 src, ++ struct jit_ctx *ctx, const bool is64, ++ const bool hi, const u8 op) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ s8 rn, rd; ++ ++ rn = arm_bpf_get_reg32(src, tmp[1], ctx); ++ rd = arm_bpf_get_reg32(dst, tmp[0], ctx); ++ /* ALU operation */ ++ emit_alu_r(rd, rn, is64, hi, op, ctx); ++ arm_bpf_put_reg32(dst, rd, ctx); ++} ++ ++/* ALU operation (64 bit) */ ++static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], ++ const s8 src[], struct jit_ctx *ctx, ++ const u8 op) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ if (is64) { ++ const s8 *rs; ++ ++ rs = arm_bpf_get_reg64(src, tmp2, ctx); ++ ++ /* ALU operation */ ++ emit_alu_r(rd[1], rs[1], true, false, op, ctx); ++ emit_alu_r(rd[0], rs[0], true, true, op, ctx); + } else { +- _emit(cond, ARM_MOV_I(ARM_R0, 0), ctx); +- _emit(cond, ARM_B(b_imm(ctx->skf->len, ctx)), ctx); ++ s8 rs; ++ ++ rs = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ ++ /* ALU operation */ ++ emit_alu_r(rd[1], rs, true, false, op, ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(rd[0], 0, ctx); + } ++ ++ arm_bpf_put_reg64(dst, rd, ctx); + } + +-static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) +-{ +-#if __LINUX_ARM_ARCH__ < 5 +- emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); ++/* dst = src (4 bytes)*/ ++static inline void emit_a32_mov_r(const s8 dst, const s8 src, ++ struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ s8 rt; ++ ++ rt = arm_bpf_get_reg32(src, tmp[0], ctx); ++ arm_bpf_put_reg32(dst, rt, ctx); ++} ++ ++/* dst = src */ ++static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], ++ const s8 src[], ++ struct jit_ctx *ctx) { ++ if (!is64) { ++ emit_a32_mov_r(dst_lo, src_lo, ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ /* Zero out high 4 bytes */ ++ emit_a32_mov_i(dst_hi, 0, ctx); ++ } else if (__LINUX_ARM_ARCH__ < 6 && ++ ctx->cpu_architecture < CPU_ARCH_ARMv5TE) { ++ /* complete 8 byte move */ ++ emit_a32_mov_r(dst_lo, src_lo, ctx); ++ emit_a32_mov_r(dst_hi, src_hi, ctx); ++ } else if (is_stacked(src_lo) && is_stacked(dst_lo)) { ++ const u8 *tmp = bpf2a32[TMP_REG_1]; ++ ++ emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); ++ emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); ++ } else if (is_stacked(src_lo)) { ++ emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); ++ } else if (is_stacked(dst_lo)) { ++ emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); ++ } else { ++ emit(ARM_MOV_R(dst[0], src[0]), ctx); ++ emit(ARM_MOV_R(dst[1], src[1]), ctx); ++ } ++} + +- if (elf_hwcap & HWCAP_THUMB) +- emit(ARM_BX(tgt_reg), ctx); +- else +- emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); +-#else +- emit(ARM_BLX_R(tgt_reg), ctx); +-#endif ++/* Shift operations */ ++static inline void emit_a32_alu_i(const s8 dst, const u32 val, ++ struct jit_ctx *ctx, const u8 op) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ s8 rd; ++ ++ rd = arm_bpf_get_reg32(dst, tmp[0], ctx); ++ ++ /* Do shift operation */ ++ switch (op) { ++ case BPF_LSH: ++ emit(ARM_LSL_I(rd, rd, val), ctx); ++ break; ++ case BPF_RSH: ++ emit(ARM_LSR_I(rd, rd, val), ctx); ++ break; ++ case BPF_NEG: ++ emit(ARM_RSB_I(rd, rd, val), ctx); ++ break; ++ } ++ ++ arm_bpf_put_reg32(dst, rd, ctx); + } + +-static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, +- int bpf_op) +-{ +-#if __LINUX_ARM_ARCH__ == 7 +- if (elf_hwcap & HWCAP_IDIVA) { +- if (bpf_op == BPF_DIV) +- emit(ARM_UDIV(rd, rm, rn), ctx); +- else { +- emit(ARM_UDIV(ARM_R3, rm, rn), ctx); +- emit(ARM_MLS(rd, rn, ARM_R3, rm), ctx); ++/* dst = ~dst (64 bit) */ ++static inline void emit_a32_neg64(const s8 dst[], ++ struct jit_ctx *ctx){ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *rd; ++ ++ /* Setup Operand */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do Negate Operation */ ++ emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); ++ emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); ++ ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++/* dst = dst << src */ ++static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], ++ struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ s8 rt; ++ ++ /* Setup Operands */ ++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do LSH operation */ ++ emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); ++ emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); ++ emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); ++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); ++ emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); ++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); ++ ++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); ++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); ++} ++ ++/* dst = dst >> src (signed)*/ ++static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], ++ struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ s8 rt; ++ ++ /* Setup Operands */ ++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do the ARSH operation */ ++ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); ++ emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); ++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); ++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); ++ _emit(ARM_COND_MI, ARM_B(0), ctx); ++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx); ++ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); ++ ++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); ++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); ++} ++ ++/* dst = dst >> src */ ++static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], ++ struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ s8 rt; ++ ++ /* Setup Operands */ ++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do RSH operation */ ++ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); ++ emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); ++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); ++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); ++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); ++ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); ++ ++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); ++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); ++} ++ ++/* dst = dst << val */ ++static inline void emit_a32_lsh_i64(const s8 dst[], ++ const u32 val, struct jit_ctx *ctx){ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ ++ /* Setup operands */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do LSH operation */ ++ if (val < 32) { ++ emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); ++ emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); ++ emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); ++ } else { ++ if (val == 32) ++ emit(ARM_MOV_R(rd[0], rd[1]), ctx); ++ else ++ emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); ++ emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); ++ } ++ ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++/* dst = dst >> val */ ++static inline void emit_a32_rsh_i64(const s8 dst[], ++ const u32 val, struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ ++ /* Setup operands */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do LSR operation */ ++ if (val == 0) { ++ /* An immediate value of 0 encodes a shift amount of 32 ++ * for LSR. To shift by 0, don't do anything. ++ */ ++ } else if (val < 32) { ++ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); ++ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); ++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); ++ } else if (val == 32) { ++ emit(ARM_MOV_R(rd[1], rd[0]), ctx); ++ emit(ARM_MOV_I(rd[0], 0), ctx); ++ } else { ++ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); ++ emit(ARM_MOV_I(rd[0], 0), ctx); ++ } ++ ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++/* dst = dst >> val (signed) */ ++static inline void emit_a32_arsh_i64(const s8 dst[], ++ const u32 val, struct jit_ctx *ctx){ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd; ++ ++ /* Setup operands */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Do ARSH operation */ ++ if (val == 0) { ++ /* An immediate value of 0 encodes a shift amount of 32 ++ * for ASR. To shift by 0, don't do anything. ++ */ ++ } else if (val < 32) { ++ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); ++ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); ++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); ++ } else if (val == 32) { ++ emit(ARM_MOV_R(rd[1], rd[0]), ctx); ++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); ++ } else { ++ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); ++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); ++ } ++ ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], ++ struct jit_ctx *ctx) { ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rd, *rt; ++ ++ /* Setup operands for multiplication */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ rt = arm_bpf_get_reg64(src, tmp2, ctx); ++ ++ /* Do Multiplication */ ++ emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); ++ emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); ++ emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); ++ ++ emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); ++ emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); ++ ++ arm_bpf_put_reg32(dst_lo, ARM_IP, ctx); ++ arm_bpf_put_reg32(dst_hi, rd[0], ctx); ++} ++ ++static bool is_ldst_imm(s16 off, const u8 size) ++{ ++ s16 off_max = 0; ++ ++ switch (size) { ++ case BPF_B: ++ case BPF_W: ++ off_max = 0xfff; ++ break; ++ case BPF_H: ++ off_max = 0xff; ++ break; ++ case BPF_DW: ++ /* Need to make sure off+4 does not overflow. */ ++ off_max = 0xfff - 4; ++ break; ++ } ++ return -off_max <= off && off <= off_max; ++} ++ ++/* *(size *)(dst + off) = src */ ++static inline void emit_str_r(const s8 dst, const s8 src[], ++ s16 off, struct jit_ctx *ctx, const u8 sz){ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ s8 rd; ++ ++ rd = arm_bpf_get_reg32(dst, tmp[1], ctx); ++ ++ if (!is_ldst_imm(off, sz)) { ++ emit_a32_mov_i(tmp[0], off, ctx); ++ emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); ++ rd = tmp[0]; ++ off = 0; ++ } ++ switch (sz) { ++ case BPF_B: ++ /* Store a Byte */ ++ emit(ARM_STRB_I(src_lo, rd, off), ctx); ++ break; ++ case BPF_H: ++ /* Store a HalfWord */ ++ emit(ARM_STRH_I(src_lo, rd, off), ctx); ++ break; ++ case BPF_W: ++ /* Store a Word */ ++ emit(ARM_STR_I(src_lo, rd, off), ctx); ++ break; ++ case BPF_DW: ++ /* Store a Double Word */ ++ emit(ARM_STR_I(src_lo, rd, off), ctx); ++ emit(ARM_STR_I(src_hi, rd, off + 4), ctx); ++ break; ++ } ++} ++ ++/* dst = *(size*)(src + off) */ ++static inline void emit_ldx_r(const s8 dst[], const s8 src, ++ s16 off, struct jit_ctx *ctx, const u8 sz){ ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *rd = is_stacked(dst_lo) ? tmp : dst; ++ s8 rm = src; ++ ++ if (!is_ldst_imm(off, sz)) { ++ emit_a32_mov_i(tmp[0], off, ctx); ++ emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); ++ rm = tmp[0]; ++ off = 0; ++ } else if (rd[1] == rm) { ++ emit(ARM_MOV_R(tmp[0], rm), ctx); ++ rm = tmp[0]; ++ } ++ switch (sz) { ++ case BPF_B: ++ /* Load a Byte */ ++ emit(ARM_LDRB_I(rd[1], rm, off), ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(rd[0], 0, ctx); ++ break; ++ case BPF_H: ++ /* Load a HalfWord */ ++ emit(ARM_LDRH_I(rd[1], rm, off), ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(rd[0], 0, ctx); ++ break; ++ case BPF_W: ++ /* Load a Word */ ++ emit(ARM_LDR_I(rd[1], rm, off), ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(rd[0], 0, ctx); ++ break; ++ case BPF_DW: ++ /* Load a Double Word */ ++ emit(ARM_LDR_I(rd[1], rm, off), ctx); ++ emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); ++ break; ++ } ++ arm_bpf_put_reg64(dst, rd, ctx); ++} ++ ++/* Arithmatic Operation */ ++static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm, ++ const u8 rn, struct jit_ctx *ctx, u8 op, ++ bool is_jmp64) { ++ switch (op) { ++ case BPF_JSET: ++ if (is_jmp64) { ++ emit(ARM_AND_R(ARM_IP, rt, rn), ctx); ++ emit(ARM_AND_R(ARM_LR, rd, rm), ctx); ++ emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx); ++ } else { ++ emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx); + } +- return; ++ break; ++ case BPF_JEQ: ++ case BPF_JNE: ++ case BPF_JGT: ++ case BPF_JGE: ++ case BPF_JLE: ++ case BPF_JLT: ++ if (is_jmp64) { ++ emit(ARM_CMP_R(rd, rm), ctx); ++ /* Only compare low halve if high halve are equal. */ ++ _emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx); ++ } else { ++ emit(ARM_CMP_R(rt, rn), ctx); ++ } ++ break; ++ case BPF_JSLE: ++ case BPF_JSGT: ++ emit(ARM_CMP_R(rn, rt), ctx); ++ if (is_jmp64) ++ emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx); ++ break; ++ case BPF_JSLT: ++ case BPF_JSGE: ++ emit(ARM_CMP_R(rt, rn), ctx); ++ if (is_jmp64) ++ emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx); ++ break; + } +-#endif ++} + +- /* +- * For BPF_ALU | BPF_DIV | BPF_K instructions, rm is ARM_R4 +- * (r_A) and rn is ARM_R0 (r_scratch) so load rn first into +- * ARM_R1 to avoid accidentally overwriting ARM_R0 with rm +- * before using it as a source for ARM_R1. +- * +- * For BPF_ALU | BPF_DIV | BPF_X rm is ARM_R4 (r_A) and rn is +- * ARM_R5 (r_X) so there is no particular register overlap +- * issues. ++static int out_offset = -1; /* initialized on the first pass of build_body() */ ++static int emit_bpf_tail_call(struct jit_ctx *ctx) ++{ ++ ++ /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ ++ const s8 *r2 = bpf2a32[BPF_REG_2]; ++ const s8 *r3 = bpf2a32[BPF_REG_3]; ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *tcc = bpf2a32[TCALL_CNT]; ++ const s8 *tc; ++ const int idx0 = ctx->idx; ++#define cur_offset (ctx->idx - idx0) ++#define jmp_offset (out_offset - (cur_offset) - 2) ++ u32 lo, hi; ++ s8 r_array, r_index; ++ int off; ++ ++ /* if (index >= array->map.max_entries) ++ * goto out; + */ +- if (rn != ARM_R1) +- emit(ARM_MOV_R(ARM_R1, rn), ctx); +- if (rm != ARM_R0) +- emit(ARM_MOV_R(ARM_R0, rm), ctx); ++ BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) > ++ ARM_INST_LDST__IMM12); ++ off = offsetof(struct bpf_array, map.max_entries); ++ r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx); ++ /* index is 32-bit for arrays */ ++ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); ++ /* array->map.max_entries */ ++ emit(ARM_LDR_I(tmp[1], r_array, off), ctx); ++ /* index >= array->map.max_entries */ ++ emit(ARM_CMP_R(r_index, tmp[1]), ctx); ++ _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); ++ ++ /* tmp2[0] = array, tmp2[1] = index */ ++ ++ /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) ++ * goto out; ++ * tail_call_cnt++; ++ */ ++ lo = (u32)MAX_TAIL_CALL_CNT; ++ hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32); ++ tc = arm_bpf_get_reg64(tcc, tmp, ctx); ++ emit(ARM_CMP_I(tc[0], hi), ctx); ++ _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx); ++ _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); ++ emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); ++ emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); ++ arm_bpf_put_reg64(tcc, tmp, ctx); ++ ++ /* prog = array->ptrs[index] ++ * if (prog == NULL) ++ * goto out; ++ */ ++ BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0); ++ off = imm8m(offsetof(struct bpf_array, ptrs)); ++ emit(ARM_ADD_I(tmp[1], r_array, off), ctx); ++ emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); ++ emit(ARM_CMP_I(tmp[1], 0), ctx); ++ _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); ++ ++ /* goto *(prog->bpf_func + prologue_size); */ ++ BUILD_BUG_ON(offsetof(struct bpf_prog, bpf_func) > ++ ARM_INST_LDST__IMM12); ++ off = offsetof(struct bpf_prog, bpf_func); ++ emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); ++ emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); ++ emit_bx_r(tmp[1], ctx); ++ ++ /* out: */ ++ if (out_offset == -1) ++ out_offset = cur_offset; ++ if (cur_offset != out_offset) { ++ pr_err_once("tail_call out_offset = %d, expected %d!\n", ++ cur_offset, out_offset); ++ return -1; ++ } ++ return 0; ++#undef cur_offset ++#undef jmp_offset ++} ++ ++/* 0xabcd => 0xcdab */ ++static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx) ++{ ++#if __LINUX_ARM_ARCH__ < 6 ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; + +- ctx->seen |= SEEN_CALL; +- emit_mov_i(ARM_R3, bpf_op == BPF_DIV ? (u32)jit_udiv : (u32)jit_mod, +- ctx); +- emit_blx_r(ARM_R3, ctx); ++ emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); ++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); ++ emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); ++ emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx); ++#else /* ARMv6+ */ ++ emit(ARM_REV16(rd, rn), ctx); ++#endif ++} + +- if (rd != ARM_R0) +- emit(ARM_MOV_R(rd, ARM_R0), ctx); ++/* 0xabcdefgh => 0xghefcdab */ ++static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) ++{ ++#if __LINUX_ARM_ARCH__ < 6 ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ ++ emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); ++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); ++ emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx); ++ ++ emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx); ++ emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx); ++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx); ++ emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); ++ emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx); ++ emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx); ++ emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx); ++ ++#else /* ARMv6+ */ ++ emit(ARM_REV(rd, rn), ctx); ++#endif + } + +-static inline void update_on_xread(struct jit_ctx *ctx) ++// push the scratch stack register on top of the stack ++static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx) + { +- if (!(ctx->seen & SEEN_X)) +- ctx->flags |= FLAG_NEED_X_RESET; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s8 *rt; ++ u16 reg_set = 0; ++ ++ rt = arm_bpf_get_reg64(src, tmp2, ctx); + +- ctx->seen |= SEEN_X; ++ reg_set = (1 << rt[1]) | (1 << rt[0]); ++ emit(ARM_PUSH(reg_set), ctx); + } + +-static int build_body(struct jit_ctx *ctx) ++static void build_prologue(struct jit_ctx *ctx) + { +- void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w}; +- const struct bpf_prog *prog = ctx->skf; +- const struct sock_filter *inst; +- unsigned i, load_order, off, condt; +- int imm12; +- u32 k; ++ const s8 r0 = bpf2a32[BPF_REG_0][1]; ++ const s8 r2 = bpf2a32[BPF_REG_1][1]; ++ const s8 r3 = bpf2a32[BPF_REG_1][0]; ++ const s8 r4 = bpf2a32[BPF_REG_6][1]; ++ const s8 fplo = bpf2a32[BPF_REG_FP][1]; ++ const s8 fphi = bpf2a32[BPF_REG_FP][0]; ++ const s8 *tcc = bpf2a32[TCALL_CNT]; + +- for (i = 0; i < prog->len; i++) { +- u16 code; ++ /* Save callee saved registers. */ ++#ifdef CONFIG_FRAME_POINTER ++ u16 reg_set = CALLEE_PUSH_MASK | 1 << ARM_IP | 1 << ARM_PC; ++ emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); ++ emit(ARM_PUSH(reg_set), ctx); ++ emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); ++#else ++ emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx); ++ emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx); ++#endif ++ /* Save frame pointer for later */ ++ emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx); + +- inst = &(prog->insns[i]); +- /* K as an immediate value operand */ +- k = inst->k; +- code = bpf_anc_helper(inst); ++ ctx->stack_size = imm8m(STACK_SIZE); + +- /* compute offsets only in the fake pass */ +- if (ctx->target == NULL) +- ctx->offsets[i] = ctx->idx * 4; ++ /* Set up function call stack */ ++ emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx); + +- switch (code) { +- case BPF_LD | BPF_IMM: +- emit_mov_i(r_A, k, ctx); +- break; +- case BPF_LD | BPF_W | BPF_LEN: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); +- emit(ARM_LDR_I(r_A, r_skb, +- offsetof(struct sk_buff, len)), ctx); +- break; +- case BPF_LD | BPF_MEM: +- /* A = scratch[k] */ +- ctx->seen |= SEEN_MEM_WORD(k); +- emit(ARM_LDR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); +- break; +- case BPF_LD | BPF_W | BPF_ABS: +- load_order = 2; +- goto load; +- case BPF_LD | BPF_H | BPF_ABS: +- load_order = 1; +- goto load; +- case BPF_LD | BPF_B | BPF_ABS: +- load_order = 0; +-load: +- emit_mov_i(r_off, k, ctx); +-load_common: +- ctx->seen |= SEEN_DATA | SEEN_CALL; +- +- if (load_order > 0) { +- emit(ARM_SUB_I(r_scratch, r_skb_hl, +- 1 << load_order), ctx); +- emit(ARM_CMP_R(r_scratch, r_off), ctx); +- condt = ARM_COND_GE; +- } else { +- emit(ARM_CMP_R(r_skb_hl, r_off), ctx); +- condt = ARM_COND_HI; +- } ++ /* Set up BPF prog stack base register */ ++ emit_a32_mov_r(fplo, ARM_IP, ctx); ++ emit_a32_mov_i(fphi, 0, ctx); + +- /* +- * test for negative offset, only if we are +- * currently scheduled to take the fast +- * path. this will update the flags so that +- * the slowpath instruction are ignored if the +- * offset is negative. +- * +- * for loard_order == 0 the HI condition will +- * make loads at offset 0 take the slow path too. +- */ +- _emit(condt, ARM_CMP_I(r_off, 0), ctx); ++ /* mov r4, 0 */ ++ emit(ARM_MOV_I(r4, 0), ctx); + +- _emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data), +- ctx); ++ /* Move BPF_CTX to BPF_R1 */ ++ emit(ARM_MOV_R(r3, r4), ctx); ++ emit(ARM_MOV_R(r2, r0), ctx); ++ /* Initialize Tail Count */ ++ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx); ++ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx); ++ /* end of prologue */ ++} + +- if (load_order == 0) +- _emit(condt, ARM_LDRB_I(r_A, r_scratch, 0), +- ctx); +- else if (load_order == 1) +- emit_load_be16(condt, r_A, r_scratch, ctx); +- else if (load_order == 2) +- emit_load_be32(condt, r_A, r_scratch, ctx); +- +- _emit(condt, ARM_B(b_imm(i + 1, ctx)), ctx); +- +- /* the slowpath */ +- emit_mov_i(ARM_R3, (u32)load_func[load_order], ctx); +- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); +- /* the offset is already in R1 */ +- emit_blx_r(ARM_R3, ctx); +- /* check the result of skb_copy_bits */ +- emit(ARM_CMP_I(ARM_R1, 0), ctx); +- emit_err_ret(ARM_COND_NE, ctx); +- emit(ARM_MOV_R(r_A, ARM_R0), ctx); +- break; +- case BPF_LD | BPF_W | BPF_IND: +- load_order = 2; +- goto load_ind; +- case BPF_LD | BPF_H | BPF_IND: +- load_order = 1; +- goto load_ind; +- case BPF_LD | BPF_B | BPF_IND: +- load_order = 0; +-load_ind: +- update_on_xread(ctx); +- OP_IMM3(ARM_ADD, r_off, r_X, k, ctx); +- goto load_common; +- case BPF_LDX | BPF_IMM: +- ctx->seen |= SEEN_X; +- emit_mov_i(r_X, k, ctx); +- break; +- case BPF_LDX | BPF_W | BPF_LEN: +- ctx->seen |= SEEN_X | SEEN_SKB; +- emit(ARM_LDR_I(r_X, r_skb, +- offsetof(struct sk_buff, len)), ctx); +- break; +- case BPF_LDX | BPF_MEM: +- ctx->seen |= SEEN_X | SEEN_MEM_WORD(k); +- emit(ARM_LDR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); +- break; +- case BPF_LDX | BPF_B | BPF_MSH: +- /* x = ((*(frame + k)) & 0xf) << 2; */ +- ctx->seen |= SEEN_X | SEEN_DATA | SEEN_CALL; +- /* the interpreter should deal with the negative K */ +- if ((int)k < 0) +- return -1; +- /* offset in r1: we might have to take the slow path */ +- emit_mov_i(r_off, k, ctx); +- emit(ARM_CMP_R(r_skb_hl, r_off), ctx); +- +- /* load in r0: common with the slowpath */ +- _emit(ARM_COND_HI, ARM_LDRB_R(ARM_R0, r_skb_data, +- ARM_R1), ctx); +- /* +- * emit_mov_i() might generate one or two instructions, +- * the same holds for emit_blx_r() +- */ +- _emit(ARM_COND_HI, ARM_B(b_imm(i + 1, ctx) - 2), ctx); ++/* restore callee saved registers. */ ++static void build_epilogue(struct jit_ctx *ctx) ++{ ++#ifdef CONFIG_FRAME_POINTER ++ /* When using frame pointers, some additional registers need to ++ * be loaded. */ ++ u16 reg_set = CALLEE_POP_MASK | 1 << ARM_SP; ++ emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx); ++ emit(ARM_LDM(ARM_SP, reg_set), ctx); ++#else ++ /* Restore callee saved registers. */ ++ emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx); ++ emit(ARM_POP(CALLEE_POP_MASK), ctx); ++#endif ++} + +- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); +- /* r_off is r1 */ +- emit_mov_i(ARM_R3, (u32)jit_get_skb_b, ctx); +- emit_blx_r(ARM_R3, ctx); +- /* check the return value of skb_copy_bits */ +- emit(ARM_CMP_I(ARM_R1, 0), ctx); +- emit_err_ret(ARM_COND_NE, ctx); +- +- emit(ARM_AND_I(r_X, ARM_R0, 0x00f), ctx); +- emit(ARM_LSL_I(r_X, r_X, 2), ctx); +- break; +- case BPF_ST: +- ctx->seen |= SEEN_MEM_WORD(k); +- emit(ARM_STR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); +- break; +- case BPF_STX: +- update_on_xread(ctx); +- ctx->seen |= SEEN_MEM_WORD(k); +- emit(ARM_STR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); +- break; +- case BPF_ALU | BPF_ADD | BPF_K: +- /* A += K */ +- OP_IMM3(ARM_ADD, r_A, r_A, k, ctx); +- break; +- case BPF_ALU | BPF_ADD | BPF_X: +- update_on_xread(ctx); +- emit(ARM_ADD_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_SUB | BPF_K: +- /* A -= K */ +- OP_IMM3(ARM_SUB, r_A, r_A, k, ctx); +- break; +- case BPF_ALU | BPF_SUB | BPF_X: +- update_on_xread(ctx); +- emit(ARM_SUB_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_MUL | BPF_K: +- /* A *= K */ +- emit_mov_i(r_scratch, k, ctx); +- emit(ARM_MUL(r_A, r_A, r_scratch), ctx); +- break; +- case BPF_ALU | BPF_MUL | BPF_X: +- update_on_xread(ctx); +- emit(ARM_MUL(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_DIV | BPF_K: +- if (k == 1) +- break; +- emit_mov_i(r_scratch, k, ctx); +- emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_DIV); +- break; +- case BPF_ALU | BPF_DIV | BPF_X: +- update_on_xread(ctx); +- emit(ARM_CMP_I(r_X, 0), ctx); +- emit_err_ret(ARM_COND_EQ, ctx); +- emit_udivmod(r_A, r_A, r_X, ctx, BPF_DIV); +- break; +- case BPF_ALU | BPF_MOD | BPF_K: +- if (k == 1) { +- emit_mov_i(r_A, 0, ctx); ++/* ++ * Convert an eBPF instruction to native instruction, i.e ++ * JITs an eBPF instruction. ++ * Returns : ++ * 0 - Successfully JITed an 8-byte eBPF instruction ++ * >0 - Successfully JITed a 16-byte eBPF instruction ++ * <0 - Failed to JIT. ++ */ ++static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) ++{ ++ const u8 code = insn->code; ++ const s8 *dst = bpf2a32[insn->dst_reg]; ++ const s8 *src = bpf2a32[insn->src_reg]; ++ const s8 *tmp = bpf2a32[TMP_REG_1]; ++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; ++ const s16 off = insn->off; ++ const s32 imm = insn->imm; ++ const int i = insn - ctx->prog->insnsi; ++ const bool is64 = BPF_CLASS(code) == BPF_ALU64; ++ const s8 *rd, *rs; ++ s8 rd_lo, rt, rm, rn; ++ s32 jmp_offset; ++ ++#define check_imm(bits, imm) do { \ ++ if ((imm) >= (1 << ((bits) - 1)) || \ ++ (imm) < -(1 << ((bits) - 1))) { \ ++ pr_info("[%2d] imm=%d(0x%x) out of range\n", \ ++ i, imm, imm); \ ++ return -EINVAL; \ ++ } \ ++} while (0) ++#define check_imm24(imm) check_imm(24, imm) ++ ++ switch (code) { ++ /* ALU operations */ ++ ++ /* dst = src */ ++ case BPF_ALU | BPF_MOV | BPF_K: ++ case BPF_ALU | BPF_MOV | BPF_X: ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ case BPF_ALU64 | BPF_MOV | BPF_X: ++ switch (BPF_SRC(code)) { ++ case BPF_X: ++ if (imm == 1) { ++ /* Special mov32 for zext */ ++ emit_a32_mov_i(dst_hi, 0, ctx); + break; + } +- emit_mov_i(r_scratch, k, ctx); +- emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_MOD); ++ emit_a32_mov_r64(is64, dst, src, ctx); + break; +- case BPF_ALU | BPF_MOD | BPF_X: +- update_on_xread(ctx); +- emit(ARM_CMP_I(r_X, 0), ctx); +- emit_err_ret(ARM_COND_EQ, ctx); +- emit_udivmod(r_A, r_A, r_X, ctx, BPF_MOD); +- break; +- case BPF_ALU | BPF_OR | BPF_K: +- /* A |= K */ +- OP_IMM3(ARM_ORR, r_A, r_A, k, ctx); +- break; +- case BPF_ALU | BPF_OR | BPF_X: +- update_on_xread(ctx); +- emit(ARM_ORR_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_XOR | BPF_K: +- /* A ^= K; */ +- OP_IMM3(ARM_EOR, r_A, r_A, k, ctx); +- break; +- case BPF_ANC | SKF_AD_ALU_XOR_X: +- case BPF_ALU | BPF_XOR | BPF_X: +- /* A ^= X */ +- update_on_xread(ctx); +- emit(ARM_EOR_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_AND | BPF_K: +- /* A &= K */ +- OP_IMM3(ARM_AND, r_A, r_A, k, ctx); +- break; +- case BPF_ALU | BPF_AND | BPF_X: +- update_on_xread(ctx); +- emit(ARM_AND_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_LSH | BPF_K: +- if (unlikely(k > 31)) +- return -1; +- emit(ARM_LSL_I(r_A, r_A, k), ctx); +- break; +- case BPF_ALU | BPF_LSH | BPF_X: +- update_on_xread(ctx); +- emit(ARM_LSL_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_RSH | BPF_K: +- if (unlikely(k > 31)) +- return -1; +- if (k) +- emit(ARM_LSR_I(r_A, r_A, k), ctx); +- break; +- case BPF_ALU | BPF_RSH | BPF_X: +- update_on_xread(ctx); +- emit(ARM_LSR_R(r_A, r_A, r_X), ctx); +- break; +- case BPF_ALU | BPF_NEG: +- /* A = -A */ +- emit(ARM_RSB_I(r_A, r_A, 0), ctx); +- break; +- case BPF_JMP | BPF_JA: +- /* pc += K */ +- emit(ARM_B(b_imm(i + k + 1, ctx)), ctx); +- break; +- case BPF_JMP | BPF_JEQ | BPF_K: +- /* pc += (A == K) ? pc->jt : pc->jf */ +- condt = ARM_COND_EQ; +- goto cmp_imm; +- case BPF_JMP | BPF_JGT | BPF_K: +- /* pc += (A > K) ? pc->jt : pc->jf */ +- condt = ARM_COND_HI; +- goto cmp_imm; +- case BPF_JMP | BPF_JGE | BPF_K: +- /* pc += (A >= K) ? pc->jt : pc->jf */ +- condt = ARM_COND_HS; +-cmp_imm: +- imm12 = imm8m(k); +- if (imm12 < 0) { +- emit_mov_i_no8m(r_scratch, k, ctx); +- emit(ARM_CMP_R(r_A, r_scratch), ctx); +- } else { +- emit(ARM_CMP_I(r_A, imm12), ctx); +- } +-cond_jump: +- if (inst->jt) +- _emit(condt, ARM_B(b_imm(i + inst->jt + 1, +- ctx)), ctx); +- if (inst->jf) +- _emit(condt ^ 1, ARM_B(b_imm(i + inst->jf + 1, +- ctx)), ctx); +- break; +- case BPF_JMP | BPF_JEQ | BPF_X: +- /* pc += (A == X) ? pc->jt : pc->jf */ +- condt = ARM_COND_EQ; +- goto cmp_x; +- case BPF_JMP | BPF_JGT | BPF_X: +- /* pc += (A > X) ? pc->jt : pc->jf */ +- condt = ARM_COND_HI; +- goto cmp_x; +- case BPF_JMP | BPF_JGE | BPF_X: +- /* pc += (A >= X) ? pc->jt : pc->jf */ +- condt = ARM_COND_CS; +-cmp_x: +- update_on_xread(ctx); +- emit(ARM_CMP_R(r_A, r_X), ctx); +- goto cond_jump; +- case BPF_JMP | BPF_JSET | BPF_K: +- /* pc += (A & K) ? pc->jt : pc->jf */ +- condt = ARM_COND_NE; +- /* not set iff all zeroes iff Z==1 iff EQ */ +- +- imm12 = imm8m(k); +- if (imm12 < 0) { +- emit_mov_i_no8m(r_scratch, k, ctx); +- emit(ARM_TST_R(r_A, r_scratch), ctx); +- } else { +- emit(ARM_TST_I(r_A, imm12), ctx); +- } +- goto cond_jump; +- case BPF_JMP | BPF_JSET | BPF_X: +- /* pc += (A & X) ? pc->jt : pc->jf */ +- update_on_xread(ctx); +- condt = ARM_COND_NE; +- emit(ARM_TST_R(r_A, r_X), ctx); +- goto cond_jump; +- case BPF_RET | BPF_A: +- emit(ARM_MOV_R(ARM_R0, r_A), ctx); +- goto b_epilogue; +- case BPF_RET | BPF_K: +- if ((k == 0) && (ctx->ret0_fp_idx < 0)) +- ctx->ret0_fp_idx = i; +- emit_mov_i(ARM_R0, k, ctx); +-b_epilogue: +- if (i != ctx->skf->len - 1) +- emit(ARM_B(b_imm(prog->len, ctx)), ctx); +- break; +- case BPF_MISC | BPF_TAX: +- /* X = A */ +- ctx->seen |= SEEN_X; +- emit(ARM_MOV_R(r_X, r_A), ctx); +- break; +- case BPF_MISC | BPF_TXA: +- /* A = X */ +- update_on_xread(ctx); +- emit(ARM_MOV_R(r_A, r_X), ctx); +- break; +- case BPF_ANC | SKF_AD_PROTOCOL: +- /* A = ntohs(skb->protocol) */ +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, +- protocol) != 2); +- off = offsetof(struct sk_buff, protocol); +- emit(ARM_LDRH_I(r_scratch, r_skb, off), ctx); +- emit_swap16(r_A, r_scratch, ctx); +- break; +- case BPF_ANC | SKF_AD_CPU: +- /* r_scratch = current_thread_info() */ +- OP_IMM3(ARM_BIC, r_scratch, ARM_SP, THREAD_SIZE - 1, ctx); +- /* A = current_thread_info()->cpu */ +- BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, cpu) != 4); +- off = offsetof(struct thread_info, cpu); +- emit(ARM_LDR_I(r_A, r_scratch, off), ctx); +- break; +- case BPF_ANC | SKF_AD_IFINDEX: +- case BPF_ANC | SKF_AD_HATYPE: +- /* A = skb->dev->ifindex */ +- /* A = skb->dev->type */ +- ctx->seen |= SEEN_SKB; +- off = offsetof(struct sk_buff, dev); +- emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); +- +- emit(ARM_CMP_I(r_scratch, 0), ctx); +- emit_err_ret(ARM_COND_EQ, ctx); +- +- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, +- ifindex) != 4); +- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, +- type) != 2); +- +- if (code == (BPF_ANC | SKF_AD_IFINDEX)) { +- off = offsetof(struct net_device, ifindex); +- emit(ARM_LDR_I(r_A, r_scratch, off), ctx); +- } else { +- /* +- * offset of field "type" in "struct +- * net_device" is above what can be +- * used in the ldrh rd, [rn, #imm] +- * instruction, so load the offset in +- * a register and use ldrh rd, [rn, rm] +- */ +- off = offsetof(struct net_device, type); +- emit_mov_i(ARM_R3, off, ctx); +- emit(ARM_LDRH_R(r_A, r_scratch, ARM_R3), ctx); +- } ++ case BPF_K: ++ /* Sign-extend immediate value to destination reg */ ++ emit_a32_mov_se_i64(is64, dst, imm, ctx); + break; +- case BPF_ANC | SKF_AD_MARK: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); +- off = offsetof(struct sk_buff, mark); +- emit(ARM_LDR_I(r_A, r_skb, off), ctx); +- break; +- case BPF_ANC | SKF_AD_RXHASH: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); +- off = offsetof(struct sk_buff, hash); +- emit(ARM_LDR_I(r_A, r_skb, off), ctx); +- break; +- case BPF_ANC | SKF_AD_VLAN_TAG: +- case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); +- off = offsetof(struct sk_buff, vlan_tci); +- emit(ARM_LDRH_I(r_A, r_skb, off), ctx); +- if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) +- OP_IMM3(ARM_AND, r_A, r_A, ~VLAN_TAG_PRESENT, ctx); +- else { +- OP_IMM3(ARM_LSR, r_A, r_A, 12, ctx); +- OP_IMM3(ARM_AND, r_A, r_A, 0x1, ctx); +- } ++ } ++ break; ++ /* dst = dst + src/imm */ ++ /* dst = dst - src/imm */ ++ /* dst = dst | src/imm */ ++ /* dst = dst & src/imm */ ++ /* dst = dst ^ src/imm */ ++ /* dst = dst * src/imm */ ++ /* dst = dst << src */ ++ /* dst = dst >> src */ ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_ADD | BPF_X: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_X: ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_OR | BPF_X: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_X: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_X: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_X: ++ case BPF_ALU | BPF_LSH | BPF_X: ++ case BPF_ALU | BPF_RSH | BPF_X: ++ case BPF_ALU | BPF_ARSH | BPF_K: ++ case BPF_ALU | BPF_ARSH | BPF_X: ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_ADD | BPF_X: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_X: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_X: ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_AND | BPF_X: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_X: ++ switch (BPF_SRC(code)) { ++ case BPF_X: ++ emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code)); ++ break; ++ case BPF_K: ++ /* Move immediate value to the temporary register ++ * and then do the ALU operation on the temporary ++ * register as this will sign-extend the immediate ++ * value into temporary reg and then it would be ++ * safe to do the operation on it. ++ */ ++ emit_a32_mov_se_i64(is64, tmp2, imm, ctx); ++ emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code)); + break; +- case BPF_ANC | SKF_AD_PKTTYPE: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, +- __pkt_type_offset[0]) != 1); +- off = PKT_TYPE_OFFSET(); +- emit(ARM_LDRB_I(r_A, r_skb, off), ctx); +- emit(ARM_AND_I(r_A, r_A, PKT_TYPE_MAX), ctx); +-#ifdef __BIG_ENDIAN_BITFIELD +- emit(ARM_LSR_I(r_A, r_A, 5), ctx); +-#endif ++ } ++ break; ++ /* dst = dst / src(imm) */ ++ /* dst = dst % src(imm) */ ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_X: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_X: ++ rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx); ++ switch (BPF_SRC(code)) { ++ case BPF_X: ++ rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx); ++ break; ++ case BPF_K: ++ rt = tmp2[0]; ++ emit_a32_mov_i(rt, imm, ctx); + break; +- case BPF_ANC | SKF_AD_QUEUE: +- ctx->seen |= SEEN_SKB; +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, +- queue_mapping) != 2); +- BUILD_BUG_ON(offsetof(struct sk_buff, +- queue_mapping) > 0xff); +- off = offsetof(struct sk_buff, queue_mapping); +- emit(ARM_LDRH_I(r_A, r_skb, off), ctx); +- break; +- case BPF_ANC | SKF_AD_PAY_OFFSET: +- ctx->seen |= SEEN_SKB | SEEN_CALL; +- +- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); +- emit_mov_i(ARM_R3, (unsigned int)skb_get_poff, ctx); +- emit_blx_r(ARM_R3, ctx); +- emit(ARM_MOV_R(r_A, ARM_R0), ctx); +- break; +- case BPF_LDX | BPF_W | BPF_ABS: +- /* +- * load a 32bit word from struct seccomp_data. +- * seccomp_check_filter() will already have checked +- * that k is 32bit aligned and lies within the +- * struct seccomp_data. ++ default: ++ rt = src_lo; ++ break; ++ } ++ emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code)); ++ arm_bpf_put_reg32(dst_lo, rd_lo, ctx); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(dst_hi, 0, ctx); ++ break; ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_X: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_X: ++ goto notyet; ++ /* dst = dst >> imm */ ++ /* dst = dst << imm */ ++ case BPF_ALU | BPF_RSH | BPF_K: ++ case BPF_ALU | BPF_LSH | BPF_K: ++ if (unlikely(imm > 31)) ++ return -EINVAL; ++ if (imm) ++ emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code)); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(dst_hi, 0, ctx); ++ break; ++ /* dst = dst << imm */ ++ case BPF_ALU64 | BPF_LSH | BPF_K: ++ if (unlikely(imm > 63)) ++ return -EINVAL; ++ emit_a32_lsh_i64(dst, imm, ctx); ++ break; ++ /* dst = dst >> imm */ ++ case BPF_ALU64 | BPF_RSH | BPF_K: ++ if (unlikely(imm > 63)) ++ return -EINVAL; ++ emit_a32_rsh_i64(dst, imm, ctx); ++ break; ++ /* dst = dst << src */ ++ case BPF_ALU64 | BPF_LSH | BPF_X: ++ emit_a32_lsh_r64(dst, src, ctx); ++ break; ++ /* dst = dst >> src */ ++ case BPF_ALU64 | BPF_RSH | BPF_X: ++ emit_a32_rsh_r64(dst, src, ctx); ++ break; ++ /* dst = dst >> src (signed) */ ++ case BPF_ALU64 | BPF_ARSH | BPF_X: ++ emit_a32_arsh_r64(dst, src, ctx); ++ break; ++ /* dst = dst >> imm (signed) */ ++ case BPF_ALU64 | BPF_ARSH | BPF_K: ++ if (unlikely(imm > 63)) ++ return -EINVAL; ++ emit_a32_arsh_i64(dst, imm, ctx); ++ break; ++ /* dst = ~dst */ ++ case BPF_ALU | BPF_NEG: ++ emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code)); ++ if (!ctx->prog->aux->verifier_zext) ++ emit_a32_mov_i(dst_hi, 0, ctx); ++ break; ++ /* dst = ~dst (64 bit) */ ++ case BPF_ALU64 | BPF_NEG: ++ emit_a32_neg64(dst, ctx); ++ break; ++ /* dst = dst * src/imm */ ++ case BPF_ALU64 | BPF_MUL | BPF_X: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ switch (BPF_SRC(code)) { ++ case BPF_X: ++ emit_a32_mul_r64(dst, src, ctx); ++ break; ++ case BPF_K: ++ /* Move immediate value to the temporary register ++ * and then do the multiplication on it as this ++ * will sign-extend the immediate value into temp ++ * reg then it would be safe to do the operation ++ * on it. + */ +- ctx->seen |= SEEN_SKB; +- emit(ARM_LDR_I(r_A, r_skb, k), ctx); ++ emit_a32_mov_se_i64(is64, tmp2, imm, ctx); ++ emit_a32_mul_r64(dst, tmp2, ctx); ++ break; ++ } ++ break; ++ /* dst = htole(dst) */ ++ /* dst = htobe(dst) */ ++ case BPF_ALU | BPF_END | BPF_FROM_LE: ++ case BPF_ALU | BPF_END | BPF_FROM_BE: ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ if (BPF_SRC(code) == BPF_FROM_LE) ++ goto emit_bswap_uxt; ++ switch (imm) { ++ case 16: ++ emit_rev16(rd[1], rd[1], ctx); ++ goto emit_bswap_uxt; ++ case 32: ++ emit_rev32(rd[1], rd[1], ctx); ++ goto emit_bswap_uxt; ++ case 64: ++ emit_rev32(ARM_LR, rd[1], ctx); ++ emit_rev32(rd[1], rd[0], ctx); ++ emit(ARM_MOV_R(rd[0], ARM_LR), ctx); + break; +- default: +- return -1; + } ++ goto exit; ++emit_bswap_uxt: ++ switch (imm) { ++ case 16: ++ /* zero-extend 16 bits into 64 bits */ ++#if __LINUX_ARM_ARCH__ < 6 ++ emit_a32_mov_i(tmp2[1], 0xffff, ctx); ++ emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); ++#else /* ARMv6+ */ ++ emit(ARM_UXTH(rd[1], rd[1]), ctx); ++#endif ++ if (!ctx->prog->aux->verifier_zext) ++ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); ++ break; ++ case 32: ++ /* zero-extend 32 bits into 64 bits */ ++ if (!ctx->prog->aux->verifier_zext) ++ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); ++ break; ++ case 64: ++ /* nop */ ++ break; ++ } ++exit: ++ arm_bpf_put_reg64(dst, rd, ctx); ++ break; ++ /* dst = imm64 */ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ { ++ u64 val = (u32)imm | (u64)insn[1].imm << 32; + +- if (ctx->flags & FLAG_IMM_OVERFLOW) +- /* +- * this instruction generated an overflow when +- * trying to access the literal pool, so +- * delegate this filter to the kernel interpreter. +- */ +- return -1; ++ emit_a32_mov_i64(dst, val, ctx); ++ ++ return 1; + } ++ /* LDX: dst = *(size *)(src + off) */ ++ case BPF_LDX | BPF_MEM | BPF_W: ++ case BPF_LDX | BPF_MEM | BPF_H: ++ case BPF_LDX | BPF_MEM | BPF_B: ++ case BPF_LDX | BPF_MEM | BPF_DW: ++ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); ++ break; ++ /* ST: *(size *)(dst + off) = imm */ ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ case BPF_ST | BPF_MEM | BPF_DW: ++ switch (BPF_SIZE(code)) { ++ case BPF_DW: ++ /* Sign-extend immediate value into temp reg */ ++ emit_a32_mov_se_i64(true, tmp2, imm, ctx); ++ break; ++ case BPF_W: ++ case BPF_H: ++ case BPF_B: ++ emit_a32_mov_i(tmp2[1], imm, ctx); ++ break; ++ } ++ emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code)); ++ break; ++ /* STX XADD: lock *(u32 *)(dst + off) += src */ ++ case BPF_STX | BPF_XADD | BPF_W: ++ /* STX XADD: lock *(u64 *)(dst + off) += src */ ++ case BPF_STX | BPF_XADD | BPF_DW: ++ goto notyet; ++ /* STX: *(size *)(dst + off) = src */ ++ case BPF_STX | BPF_MEM | BPF_W: ++ case BPF_STX | BPF_MEM | BPF_H: ++ case BPF_STX | BPF_MEM | BPF_B: ++ case BPF_STX | BPF_MEM | BPF_DW: ++ rs = arm_bpf_get_reg64(src, tmp2, ctx); ++ emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code)); ++ break; ++ /* PC += off if dst == src */ ++ /* PC += off if dst > src */ ++ /* PC += off if dst >= src */ ++ /* PC += off if dst < src */ ++ /* PC += off if dst <= src */ ++ /* PC += off if dst != src */ ++ /* PC += off if dst > src (signed) */ ++ /* PC += off if dst >= src (signed) */ ++ /* PC += off if dst < src (signed) */ ++ /* PC += off if dst <= src (signed) */ ++ /* PC += off if dst & src */ ++ case BPF_JMP | BPF_JEQ | BPF_X: ++ case BPF_JMP | BPF_JGT | BPF_X: ++ case BPF_JMP | BPF_JGE | BPF_X: ++ case BPF_JMP | BPF_JNE | BPF_X: ++ case BPF_JMP | BPF_JSGT | BPF_X: ++ case BPF_JMP | BPF_JSGE | BPF_X: ++ case BPF_JMP | BPF_JSET | BPF_X: ++ case BPF_JMP | BPF_JLE | BPF_X: ++ case BPF_JMP | BPF_JLT | BPF_X: ++ case BPF_JMP | BPF_JSLT | BPF_X: ++ case BPF_JMP | BPF_JSLE | BPF_X: ++ case BPF_JMP32 | BPF_JEQ | BPF_X: ++ case BPF_JMP32 | BPF_JGT | BPF_X: ++ case BPF_JMP32 | BPF_JGE | BPF_X: ++ case BPF_JMP32 | BPF_JNE | BPF_X: ++ case BPF_JMP32 | BPF_JSGT | BPF_X: ++ case BPF_JMP32 | BPF_JSGE | BPF_X: ++ case BPF_JMP32 | BPF_JSET | BPF_X: ++ case BPF_JMP32 | BPF_JLE | BPF_X: ++ case BPF_JMP32 | BPF_JLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLT | BPF_X: ++ case BPF_JMP32 | BPF_JSLE | BPF_X: ++ /* Setup source registers */ ++ rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); ++ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); ++ goto go_jmp; ++ /* PC += off if dst == imm */ ++ /* PC += off if dst > imm */ ++ /* PC += off if dst >= imm */ ++ /* PC += off if dst < imm */ ++ /* PC += off if dst <= imm */ ++ /* PC += off if dst != imm */ ++ /* PC += off if dst > imm (signed) */ ++ /* PC += off if dst >= imm (signed) */ ++ /* PC += off if dst < imm (signed) */ ++ /* PC += off if dst <= imm (signed) */ ++ /* PC += off if dst & imm */ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ if (off == 0) ++ break; ++ rm = tmp2[0]; ++ rn = tmp2[1]; ++ /* Sign-extend immediate value */ ++ emit_a32_mov_se_i64(true, tmp2, imm, ctx); ++go_jmp: ++ /* Setup destination register */ ++ rd = arm_bpf_get_reg64(dst, tmp, ctx); ++ ++ /* Check for the condition */ ++ emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code), ++ BPF_CLASS(code) == BPF_JMP); ++ ++ /* Setup JUMP instruction */ ++ jmp_offset = bpf2a32_offset(i+off, i, ctx); ++ switch (BPF_OP(code)) { ++ case BPF_JNE: ++ case BPF_JSET: ++ _emit(ARM_COND_NE, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JEQ: ++ _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JGT: ++ _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JGE: ++ _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JSGT: ++ _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JSGE: ++ _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JLE: ++ _emit(ARM_COND_LS, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JLT: ++ _emit(ARM_COND_CC, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JSLT: ++ _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx); ++ break; ++ case BPF_JSLE: ++ _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx); ++ break; ++ } ++ break; ++ /* JMP OFF */ ++ case BPF_JMP | BPF_JA: ++ { ++ if (off == 0) ++ break; ++ jmp_offset = bpf2a32_offset(i+off, i, ctx); ++ check_imm24(jmp_offset); ++ emit(ARM_B(jmp_offset), ctx); ++ break; ++ } ++ /* tail call */ ++ case BPF_JMP | BPF_TAIL_CALL: ++ if (emit_bpf_tail_call(ctx)) ++ return -EFAULT; ++ break; ++ /* function call */ ++ case BPF_JMP | BPF_CALL: ++ { ++ const s8 *r0 = bpf2a32[BPF_REG_0]; ++ const s8 *r1 = bpf2a32[BPF_REG_1]; ++ const s8 *r2 = bpf2a32[BPF_REG_2]; ++ const s8 *r3 = bpf2a32[BPF_REG_3]; ++ const s8 *r4 = bpf2a32[BPF_REG_4]; ++ const s8 *r5 = bpf2a32[BPF_REG_5]; ++ const u32 func = (u32)__bpf_call_base + (u32)imm; ++ ++ emit_a32_mov_r64(true, r0, r1, ctx); ++ emit_a32_mov_r64(true, r1, r2, ctx); ++ emit_push_r64(r5, ctx); ++ emit_push_r64(r4, ctx); ++ emit_push_r64(r3, ctx); + +- /* compute offsets only during the first pass */ +- if (ctx->target == NULL) +- ctx->offsets[i] = ctx->idx * 4; ++ emit_a32_mov_i(tmp[1], func, ctx); ++ emit_blx_r(tmp[1], ctx); + ++ emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean ++ break; ++ } ++ /* function return */ ++ case BPF_JMP | BPF_EXIT: ++ /* Optimization: when last instruction is EXIT ++ * simply fallthrough to epilogue. ++ */ ++ if (i == ctx->prog->len - 1) ++ break; ++ jmp_offset = epilogue_offset(ctx); ++ check_imm24(jmp_offset); ++ emit(ARM_B(jmp_offset), ctx); ++ break; ++notyet: ++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); ++ return -EFAULT; ++ default: ++ pr_err_once("unknown opcode %02x\n", code); ++ return -EINVAL; ++ } ++ ++ if (ctx->flags & FLAG_IMM_OVERFLOW) ++ /* ++ * this instruction generated an overflow when ++ * trying to access the literal pool, so ++ * delegate this filter to the kernel interpreter. ++ */ ++ return -1; + return 0; + } + ++static int build_body(struct jit_ctx *ctx) ++{ ++ const struct bpf_prog *prog = ctx->prog; ++ unsigned int i; ++ ++ for (i = 0; i < prog->len; i++) { ++ const struct bpf_insn *insn = &(prog->insnsi[i]); ++ int ret; ++ ++ ret = build_insn(insn, ctx); ++ ++ /* It's used with loading the 64 bit immediate value. */ ++ if (ret > 0) { ++ i++; ++ if (ctx->target == NULL) ++ ctx->offsets[i] = ctx->idx; ++ continue; ++ } ++ ++ if (ctx->target == NULL) ++ ctx->offsets[i] = ctx->idx; ++ ++ /* If unsuccesfull, return with error code */ ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++static int validate_code(struct jit_ctx *ctx) ++{ ++ int i; ++ ++ for (i = 0; i < ctx->idx; i++) { ++ if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF)) ++ return -1; ++ } ++ ++ return 0; ++} + +-void bpf_jit_compile(struct bpf_prog *fp) ++void bpf_jit_compile(struct bpf_prog *prog) + { ++ /* Nothing to do here. We support Internal BPF. */ ++} ++ ++bool bpf_jit_needs_zext(void) ++{ ++ return true; ++} ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ++{ ++ struct bpf_prog *tmp, *orig_prog = prog; + struct bpf_binary_header *header; ++ bool tmp_blinded = false; + struct jit_ctx ctx; +- unsigned tmp_idx; +- unsigned alloc_size; +- u8 *target_ptr; ++ unsigned int tmp_idx; ++ unsigned int image_size; ++ u8 *image_ptr; + +- if (!bpf_jit_enable) +- return; ++ /* If BPF JIT was not enabled then we must fall back to ++ * the interpreter. ++ */ ++ if (!prog->jit_requested) ++ return orig_prog; + +- memset(&ctx, 0, sizeof(ctx)); +- ctx.skf = fp; +- ctx.ret0_fp_idx = -1; ++ /* If constant blinding was enabled and we failed during blinding ++ * then we must fall back to the interpreter. Otherwise, we save ++ * the new JITed code. ++ */ ++ tmp = bpf_jit_blind_constants(prog); + +- ctx.offsets = kzalloc(4 * (ctx.skf->len + 1), GFP_KERNEL); +- if (ctx.offsets == NULL) +- return; ++ if (IS_ERR(tmp)) ++ return orig_prog; ++ if (tmp != prog) { ++ tmp_blinded = true; ++ prog = tmp; ++ } + +- /* fake pass to fill in the ctx->seen */ +- if (unlikely(build_body(&ctx))) ++ memset(&ctx, 0, sizeof(ctx)); ++ ctx.prog = prog; ++ ctx.cpu_architecture = cpu_architecture(); ++ ++ /* Not able to allocate memory for offsets[] , then ++ * we must fall back to the interpreter ++ */ ++ ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL); ++ if (ctx.offsets == NULL) { ++ prog = orig_prog; + goto out; ++ } ++ ++ /* 1) fake pass to find in the length of the JITed code, ++ * to compute ctx->offsets and other context variables ++ * needed to compute final JITed code. ++ * Also, calculate random starting pointer/start of JITed code ++ * which is prefixed by random number of fault instructions. ++ * ++ * If the first pass fails then there is no chance of it ++ * being successful in the second pass, so just fall back ++ * to the interpreter. ++ */ ++ if (build_body(&ctx)) { ++ prog = orig_prog; ++ goto out_off; ++ } + + tmp_idx = ctx.idx; + build_prologue(&ctx); + ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; + ++ ctx.epilogue_offset = ctx.idx; ++ + #if __LINUX_ARM_ARCH__ < 7 + tmp_idx = ctx.idx; + build_epilogue(&ctx); +@@ -1020,64 +1941,83 @@ void bpf_jit_compile(struct bpf_prog *fp + + ctx.idx += ctx.imm_count; + if (ctx.imm_count) { +- ctx.imms = kzalloc(4 * ctx.imm_count, GFP_KERNEL); +- if (ctx.imms == NULL) +- goto out; ++ ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL); ++ if (ctx.imms == NULL) { ++ prog = orig_prog; ++ goto out_off; ++ } + } + #else +- /* there's nothing after the epilogue on ARMv7 */ ++ /* there's nothing about the epilogue on ARMv7 */ + build_epilogue(&ctx); + #endif +- alloc_size = 4 * ctx.idx; +- header = bpf_jit_binary_alloc(alloc_size, &target_ptr, +- 4, jit_fill_hole); +- if (header == NULL) +- goto out; ++ /* Now we can get the actual image size of the JITed arm code. ++ * Currently, we are not considering the THUMB-2 instructions ++ * for jit, although it can decrease the size of the image. ++ * ++ * As each arm instruction is of length 32bit, we are translating ++ * number of JITed intructions into the size required to store these ++ * JITed code. ++ */ ++ image_size = sizeof(u32) * ctx.idx; + +- ctx.target = (u32 *) target_ptr; ++ /* Now we know the size of the structure to make */ ++ header = bpf_jit_binary_alloc(image_size, &image_ptr, ++ sizeof(u32), jit_fill_hole); ++ /* Not able to allocate memory for the structure then ++ * we must fall back to the interpretation ++ */ ++ if (header == NULL) { ++ prog = orig_prog; ++ goto out_imms; ++ } ++ ++ /* 2.) Actual pass to generate final JIT code */ ++ ctx.target = (u32 *) image_ptr; + ctx.idx = 0; + + build_prologue(&ctx); ++ ++ /* If building the body of the JITed code fails somehow, ++ * we fall back to the interpretation. ++ */ + if (build_body(&ctx) < 0) { +-#if __LINUX_ARM_ARCH__ < 7 +- if (ctx.imm_count) +- kfree(ctx.imms); +-#endif ++ image_ptr = NULL; + bpf_jit_binary_free(header); +- goto out; ++ prog = orig_prog; ++ goto out_imms; + } + build_epilogue(&ctx); + ++ /* 3.) Extra pass to validate JITed Code */ ++ if (validate_code(&ctx)) { ++ image_ptr = NULL; ++ bpf_jit_binary_free(header); ++ prog = orig_prog; ++ goto out_imms; ++ } + flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx)); + ++ if (bpf_jit_enable > 1) ++ /* there are 2 passes here */ ++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); ++ ++ bpf_jit_binary_lock_ro(header); ++ prog->bpf_func = (void *)ctx.target; ++ prog->jited = 1; ++ prog->jited_len = image_size; ++ ++out_imms: + #if __LINUX_ARM_ARCH__ < 7 + if (ctx.imm_count) + kfree(ctx.imms); + #endif +- +- if (bpf_jit_enable > 1) +- /* there are 2 passes here */ +- bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); +- +- set_memory_ro((unsigned long)header, header->pages); +- fp->bpf_func = (void *)ctx.target; +- fp->jited = 1; +-out: ++out_off: + kfree(ctx.offsets); +- return; ++out: ++ if (tmp_blinded) ++ bpf_jit_prog_release_other(prog, prog == orig_prog ? ++ tmp : orig_prog); ++ return prog; + } + +-void bpf_jit_free(struct bpf_prog *fp) +-{ +- unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; +- struct bpf_binary_header *header = (void *)addr; +- +- if (!fp->jited) +- goto free_filter; +- +- set_memory_rw(addr, header->pages); +- bpf_jit_binary_free(header); +- +-free_filter: +- bpf_prog_unlock_free(fp); +-} +--- a/arch/arm/net/bpf_jit_32.h ++++ b/arch/arm/net/bpf_jit_32.h +@@ -1,16 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ + /* + * Just-In-Time compiler for BPF filters on 32bit ARM + * + * Copyright (c) 2011 Mircea Gherzan +- * +- * This program is free software; you can redistribute it and/or modify it +- * under the terms of the GNU General Public License as published by the +- * Free Software Foundation; version 2 of the License. + */ + + #ifndef PFILTER_OPCODES_ARM_H + #define PFILTER_OPCODES_ARM_H + ++/* ARM 32bit Registers */ + #define ARM_R0 0 + #define ARM_R1 1 + #define ARM_R2 2 +@@ -22,40 +20,46 @@ + #define ARM_R8 8 + #define ARM_R9 9 + #define ARM_R10 10 +-#define ARM_FP 11 +-#define ARM_IP 12 +-#define ARM_SP 13 +-#define ARM_LR 14 +-#define ARM_PC 15 +- +-#define ARM_COND_EQ 0x0 +-#define ARM_COND_NE 0x1 +-#define ARM_COND_CS 0x2 ++#define ARM_FP 11 /* Frame Pointer */ ++#define ARM_IP 12 /* Intra-procedure scratch register */ ++#define ARM_SP 13 /* Stack pointer: as load/store base reg */ ++#define ARM_LR 14 /* Link Register */ ++#define ARM_PC 15 /* Program counter */ ++ ++#define ARM_COND_EQ 0x0 /* == */ ++#define ARM_COND_NE 0x1 /* != */ ++#define ARM_COND_CS 0x2 /* unsigned >= */ + #define ARM_COND_HS ARM_COND_CS +-#define ARM_COND_CC 0x3 ++#define ARM_COND_CC 0x3 /* unsigned < */ + #define ARM_COND_LO ARM_COND_CC +-#define ARM_COND_MI 0x4 +-#define ARM_COND_PL 0x5 +-#define ARM_COND_VS 0x6 +-#define ARM_COND_VC 0x7 +-#define ARM_COND_HI 0x8 +-#define ARM_COND_LS 0x9 +-#define ARM_COND_GE 0xa +-#define ARM_COND_LT 0xb +-#define ARM_COND_GT 0xc +-#define ARM_COND_LE 0xd +-#define ARM_COND_AL 0xe ++#define ARM_COND_MI 0x4 /* < 0 */ ++#define ARM_COND_PL 0x5 /* >= 0 */ ++#define ARM_COND_VS 0x6 /* Signed Overflow */ ++#define ARM_COND_VC 0x7 /* No Signed Overflow */ ++#define ARM_COND_HI 0x8 /* unsigned > */ ++#define ARM_COND_LS 0x9 /* unsigned <= */ ++#define ARM_COND_GE 0xa /* Signed >= */ ++#define ARM_COND_LT 0xb /* Signed < */ ++#define ARM_COND_GT 0xc /* Signed > */ ++#define ARM_COND_LE 0xd /* Signed <= */ ++#define ARM_COND_AL 0xe /* None */ + + /* register shift types */ + #define SRTYPE_LSL 0 + #define SRTYPE_LSR 1 + #define SRTYPE_ASR 2 + #define SRTYPE_ROR 3 ++#define SRTYPE_ASL (SRTYPE_LSL) + + #define ARM_INST_ADD_R 0x00800000 ++#define ARM_INST_ADDS_R 0x00900000 ++#define ARM_INST_ADC_R 0x00a00000 ++#define ARM_INST_ADC_I 0x02a00000 + #define ARM_INST_ADD_I 0x02800000 ++#define ARM_INST_ADDS_I 0x02900000 + + #define ARM_INST_AND_R 0x00000000 ++#define ARM_INST_ANDS_R 0x00100000 + #define ARM_INST_AND_I 0x02000000 + + #define ARM_INST_BIC_R 0x01c00000 +@@ -71,13 +75,18 @@ + #define ARM_INST_EOR_R 0x00200000 + #define ARM_INST_EOR_I 0x02200000 + +-#define ARM_INST_LDRB_I 0x05d00000 ++#define ARM_INST_LDST__U 0x00800000 ++#define ARM_INST_LDST__IMM12 0x00000fff ++#define ARM_INST_LDRB_I 0x05500000 + #define ARM_INST_LDRB_R 0x07d00000 +-#define ARM_INST_LDRH_I 0x01d000b0 ++#define ARM_INST_LDRD_I 0x014000d0 ++#define ARM_INST_LDRH_I 0x015000b0 + #define ARM_INST_LDRH_R 0x019000b0 +-#define ARM_INST_LDR_I 0x05900000 ++#define ARM_INST_LDR_I 0x05100000 ++#define ARM_INST_LDR_R 0x07900000 + + #define ARM_INST_LDM 0x08900000 ++#define ARM_INST_LDM_IA 0x08b00000 + + #define ARM_INST_LSL_I 0x01a00000 + #define ARM_INST_LSL_R 0x01a00010 +@@ -86,6 +95,7 @@ + #define ARM_INST_LSR_R 0x01a00030 + + #define ARM_INST_MOV_R 0x01a00000 ++#define ARM_INST_MOVS_R 0x01b00000 + #define ARM_INST_MOV_I 0x03a00000 + #define ARM_INST_MOVW 0x03000000 + #define ARM_INST_MOVT 0x03400000 +@@ -96,17 +106,29 @@ + #define ARM_INST_PUSH 0x092d0000 + + #define ARM_INST_ORR_R 0x01800000 ++#define ARM_INST_ORRS_R 0x01900000 + #define ARM_INST_ORR_I 0x03800000 + + #define ARM_INST_REV 0x06bf0f30 + #define ARM_INST_REV16 0x06bf0fb0 + + #define ARM_INST_RSB_I 0x02600000 ++#define ARM_INST_RSBS_I 0x02700000 ++#define ARM_INST_RSC_I 0x02e00000 + + #define ARM_INST_SUB_R 0x00400000 ++#define ARM_INST_SUBS_R 0x00500000 ++#define ARM_INST_RSB_R 0x00600000 + #define ARM_INST_SUB_I 0x02400000 +- +-#define ARM_INST_STR_I 0x05800000 ++#define ARM_INST_SUBS_I 0x02500000 ++#define ARM_INST_SBC_I 0x02c00000 ++#define ARM_INST_SBC_R 0x00c00000 ++#define ARM_INST_SBCS_R 0x00d00000 ++ ++#define ARM_INST_STR_I 0x05000000 ++#define ARM_INST_STRB_I 0x05400000 ++#define ARM_INST_STRD_I 0x014000f0 ++#define ARM_INST_STRH_I 0x014000b0 + + #define ARM_INST_TST_R 0x01100000 + #define ARM_INST_TST_I 0x03100000 +@@ -117,6 +139,8 @@ + + #define ARM_INST_MLS 0x00600090 + ++#define ARM_INST_UXTH 0x06ff0070 ++ + /* + * Use a suitable undefined instruction to use for ARM/Thumb2 faulting. + * We need to be careful not to conflict with those used by other modules +@@ -135,11 +159,18 @@ + #define _AL3_R(op, rd, rn, rm) ((op ## _R) | (rd) << 12 | (rn) << 16 | (rm)) + /* immediate */ + #define _AL3_I(op, rd, rn, imm) ((op ## _I) | (rd) << 12 | (rn) << 16 | (imm)) ++/* register with register-shift */ ++#define _AL3_SR(inst) (inst | (1 << 4)) + + #define ARM_ADD_R(rd, rn, rm) _AL3_R(ARM_INST_ADD, rd, rn, rm) ++#define ARM_ADDS_R(rd, rn, rm) _AL3_R(ARM_INST_ADDS, rd, rn, rm) + #define ARM_ADD_I(rd, rn, imm) _AL3_I(ARM_INST_ADD, rd, rn, imm) ++#define ARM_ADDS_I(rd, rn, imm) _AL3_I(ARM_INST_ADDS, rd, rn, imm) ++#define ARM_ADC_R(rd, rn, rm) _AL3_R(ARM_INST_ADC, rd, rn, rm) ++#define ARM_ADC_I(rd, rn, imm) _AL3_I(ARM_INST_ADC, rd, rn, imm) + + #define ARM_AND_R(rd, rn, rm) _AL3_R(ARM_INST_AND, rd, rn, rm) ++#define ARM_ANDS_R(rd, rn, rm) _AL3_R(ARM_INST_ANDS, rd, rn, rm) + #define ARM_AND_I(rd, rn, imm) _AL3_I(ARM_INST_AND, rd, rn, imm) + + #define ARM_BIC_R(rd, rn, rm) _AL3_R(ARM_INST_BIC, rd, rn, rm) +@@ -155,27 +186,38 @@ + #define ARM_EOR_R(rd, rn, rm) _AL3_R(ARM_INST_EOR, rd, rn, rm) + #define ARM_EOR_I(rd, rn, imm) _AL3_I(ARM_INST_EOR, rd, rn, imm) + +-#define ARM_LDR_I(rt, rn, off) (ARM_INST_LDR_I | (rt) << 12 | (rn) << 16 \ +- | (off)) +-#define ARM_LDRB_I(rt, rn, off) (ARM_INST_LDRB_I | (rt) << 12 | (rn) << 16 \ +- | (off)) +-#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | (rt) << 12 | (rn) << 16 \ ++#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | ARM_INST_LDST__U \ ++ | (rt) << 12 | (rn) << 16 \ + | (rm)) +-#define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \ +- | (((off) & 0xf0) << 4) | ((off) & 0xf)) +-#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \ ++#define ARM_LDR_R_SI(rt, rn, rm, type, imm) \ ++ (ARM_INST_LDR_R | ARM_INST_LDST__U \ ++ | (rt) << 12 | (rn) << 16 \ ++ | (imm) << 7 | (type) << 5 | (rm)) ++#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | ARM_INST_LDST__U \ ++ | (rt) << 12 | (rn) << 16 \ ++ | (rm)) ++#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | ARM_INST_LDST__U \ ++ | (rt) << 12 | (rn) << 16 \ + | (rm)) + + #define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs)) ++#define ARM_LDM_IA(rn, regs) (ARM_INST_LDM_IA | (rn) << 16 | (regs)) + + #define ARM_LSL_R(rd, rn, rm) (_AL3_R(ARM_INST_LSL, rd, 0, rn) | (rm) << 8) + #define ARM_LSL_I(rd, rn, imm) (_AL3_I(ARM_INST_LSL, rd, 0, rn) | (imm) << 7) + + #define ARM_LSR_R(rd, rn, rm) (_AL3_R(ARM_INST_LSR, rd, 0, rn) | (rm) << 8) + #define ARM_LSR_I(rd, rn, imm) (_AL3_I(ARM_INST_LSR, rd, 0, rn) | (imm) << 7) ++#define ARM_ASR_R(rd, rn, rm) (_AL3_R(ARM_INST_ASR, rd, 0, rn) | (rm) << 8) ++#define ARM_ASR_I(rd, rn, imm) (_AL3_I(ARM_INST_ASR, rd, 0, rn) | (imm) << 7) + + #define ARM_MOV_R(rd, rm) _AL3_R(ARM_INST_MOV, rd, 0, rm) ++#define ARM_MOVS_R(rd, rm) _AL3_R(ARM_INST_MOVS, rd, 0, rm) + #define ARM_MOV_I(rd, imm) _AL3_I(ARM_INST_MOV, rd, 0, imm) ++#define ARM_MOV_SR(rd, rm, type, rs) \ ++ (_AL3_SR(ARM_MOV_R(rd, rm)) | (type) << 5 | (rs) << 8) ++#define ARM_MOV_SI(rd, rm, type, imm6) \ ++ (ARM_MOV_R(rd, rm) | (type) << 5 | (imm6) << 7) + + #define ARM_MOVW(rd, imm) \ + (ARM_INST_MOVW | ((imm) >> 12) << 16 | (rd) << 12 | ((imm) & 0x0fff)) +@@ -190,19 +232,31 @@ + + #define ARM_ORR_R(rd, rn, rm) _AL3_R(ARM_INST_ORR, rd, rn, rm) + #define ARM_ORR_I(rd, rn, imm) _AL3_I(ARM_INST_ORR, rd, rn, imm) +-#define ARM_ORR_S(rd, rn, rm, type, rs) \ +- (ARM_ORR_R(rd, rn, rm) | (type) << 5 | (rs) << 7) ++#define ARM_ORR_SR(rd, rn, rm, type, rs) \ ++ (_AL3_SR(ARM_ORR_R(rd, rn, rm)) | (type) << 5 | (rs) << 8) ++#define ARM_ORRS_R(rd, rn, rm) _AL3_R(ARM_INST_ORRS, rd, rn, rm) ++#define ARM_ORRS_SR(rd, rn, rm, type, rs) \ ++ (_AL3_SR(ARM_ORRS_R(rd, rn, rm)) | (type) << 5 | (rs) << 8) ++#define ARM_ORR_SI(rd, rn, rm, type, imm6) \ ++ (ARM_ORR_R(rd, rn, rm) | (type) << 5 | (imm6) << 7) ++#define ARM_ORRS_SI(rd, rn, rm, type, imm6) \ ++ (ARM_ORRS_R(rd, rn, rm) | (type) << 5 | (imm6) << 7) + + #define ARM_REV(rd, rm) (ARM_INST_REV | (rd) << 12 | (rm)) + #define ARM_REV16(rd, rm) (ARM_INST_REV16 | (rd) << 12 | (rm)) + + #define ARM_RSB_I(rd, rn, imm) _AL3_I(ARM_INST_RSB, rd, rn, imm) ++#define ARM_RSBS_I(rd, rn, imm) _AL3_I(ARM_INST_RSBS, rd, rn, imm) ++#define ARM_RSC_I(rd, rn, imm) _AL3_I(ARM_INST_RSC, rd, rn, imm) + + #define ARM_SUB_R(rd, rn, rm) _AL3_R(ARM_INST_SUB, rd, rn, rm) ++#define ARM_SUBS_R(rd, rn, rm) _AL3_R(ARM_INST_SUBS, rd, rn, rm) ++#define ARM_RSB_R(rd, rn, rm) _AL3_R(ARM_INST_RSB, rd, rn, rm) ++#define ARM_SBC_R(rd, rn, rm) _AL3_R(ARM_INST_SBC, rd, rn, rm) ++#define ARM_SBCS_R(rd, rn, rm) _AL3_R(ARM_INST_SBCS, rd, rn, rm) + #define ARM_SUB_I(rd, rn, imm) _AL3_I(ARM_INST_SUB, rd, rn, imm) +- +-#define ARM_STR_I(rt, rn, off) (ARM_INST_STR_I | (rt) << 12 | (rn) << 16 \ +- | (off)) ++#define ARM_SUBS_I(rd, rn, imm) _AL3_I(ARM_INST_SUBS, rd, rn, imm) ++#define ARM_SBC_I(rd, rn, imm) _AL3_I(ARM_INST_SBC, rd, rn, imm) + + #define ARM_TST_R(rn, rm) _AL3_R(ARM_INST_TST, 0, rn, rm) + #define ARM_TST_I(rn, imm) _AL3_I(ARM_INST_TST, 0, rn, imm) +@@ -214,5 +268,6 @@ + + #define ARM_MLS(rd, rn, rm, ra) (ARM_INST_MLS | (rd) << 16 | (rn) | (rm) << 8 \ + | (ra) << 12) ++#define ARM_UXTH(rd, rm) (ARM_INST_UXTH | (rd) << 12 | (rm)) + + #endif /* PFILTER_OPCODES_ARM_H */ +--- a/arch/arm/net/Makefile ++++ b/arch/arm/net/Makefile +@@ -1,3 +1,4 @@ ++# SPDX-License-Identifier: GPL-2.0-only + # ARM-specific networking code + + obj-$(CONFIG_BPF_JIT) += bpf_jit_32.o +--- /dev/null ++++ b/include/linux/bpf-cgroup.h +@@ -0,0 +1,410 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef _BPF_CGROUP_H ++#define _BPF_CGROUP_H ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++struct sock; ++struct sockaddr; ++struct cgroup; ++struct sk_buff; ++struct bpf_map; ++struct bpf_prog; ++struct bpf_sock_ops_kern; ++struct bpf_cgroup_storage; ++struct ctl_table; ++struct ctl_table_header; ++ ++#ifdef CONFIG_CGROUP_BPF ++ ++extern struct static_key_false cgroup_bpf_enabled_key; ++#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) ++ ++DECLARE_PER_CPU(struct bpf_cgroup_storage*, ++ bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); ++ ++#define for_each_cgroup_storage_type(stype) \ ++ for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) ++ ++struct bpf_cgroup_storage_map; ++ ++struct bpf_storage_buffer { ++ struct rcu_head rcu; ++ char data[0]; ++}; ++ ++struct bpf_cgroup_storage { ++ union { ++ struct bpf_storage_buffer *buf; ++ void __percpu *percpu_buf; ++ }; ++ struct bpf_cgroup_storage_map *map; ++ struct bpf_cgroup_storage_key key; ++ struct list_head list; ++ struct rb_node node; ++ struct rcu_head rcu; ++}; ++ ++struct bpf_prog_list { ++ struct list_head node; ++ struct bpf_prog *prog; ++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; ++}; ++ ++struct bpf_prog_array; ++ ++struct cgroup_bpf { ++ /* array of effective progs in this cgroup */ ++ struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE]; ++ ++ /* attached progs to this cgroup and attach flags ++ * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will ++ * have either zero or one element ++ * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS ++ */ ++ struct list_head progs[MAX_BPF_ATTACH_TYPE]; ++ u32 flags[MAX_BPF_ATTACH_TYPE]; ++ ++ /* temp storage for effective prog array used by prog_attach/detach */ ++ struct bpf_prog_array *inactive; ++ ++ /* reference counter used to detach bpf programs after cgroup removal */ ++ struct percpu_ref refcnt; ++ ++ /* cgroup_bpf is released using a work queue */ ++ struct work_struct release_work; ++}; ++ ++int cgroup_bpf_inherit(struct cgroup *cgrp); ++void cgroup_bpf_offline(struct cgroup *cgrp); ++ ++int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type, u32 flags); ++int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type); ++int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, ++ union bpf_attr __user *uattr); ++ ++/* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */ ++int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type, u32 flags); ++int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type, u32 flags); ++int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, ++ union bpf_attr __user *uattr); ++ ++int __cgroup_bpf_run_filter_skb(struct sock *sk, ++ struct sk_buff *skb, ++ enum bpf_attach_type type); ++ ++int __cgroup_bpf_run_filter_sk(struct sock *sk, ++ enum bpf_attach_type type); ++ ++int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, ++ struct sockaddr *uaddr, ++ enum bpf_attach_type type, ++ void *t_ctx); ++ ++int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, ++ struct bpf_sock_ops_kern *sock_ops, ++ enum bpf_attach_type type); ++ ++int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, ++ short access, enum bpf_attach_type type); ++ ++int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, ++ struct ctl_table *table, int write, ++ void __user *buf, size_t *pcount, ++ loff_t *ppos, void **new_buf, ++ enum bpf_attach_type type); ++ ++int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level, ++ int *optname, char __user *optval, ++ int *optlen, char **kernel_optval); ++int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, ++ int optname, char __user *optval, ++ int __user *optlen, int max_optlen, ++ int retval); ++ ++static inline enum bpf_cgroup_storage_type cgroup_storage_type( ++ struct bpf_map *map) ++{ ++ if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) ++ return BPF_CGROUP_STORAGE_PERCPU; ++ ++ return BPF_CGROUP_STORAGE_SHARED; ++} ++ ++static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage ++ *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) ++{ ++ enum bpf_cgroup_storage_type stype; ++ ++ for_each_cgroup_storage_type(stype) ++ this_cpu_write(bpf_cgroup_storage[stype], storage[stype]); ++} ++ ++struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, ++ enum bpf_cgroup_storage_type stype); ++void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage); ++void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, ++ struct cgroup *cgroup, ++ enum bpf_attach_type type); ++void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage); ++int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *map); ++void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *map); ++ ++int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value); ++int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, ++ void *value, u64 flags); ++ ++/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */ ++#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ ++ BPF_CGROUP_INET_INGRESS); \ ++ \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled && sk && sk == skb->sk) { \ ++ typeof(sk) __sk = sk_to_full_sk(sk); \ ++ if (sk_fullsock(__sk)) \ ++ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \ ++ BPF_CGROUP_INET_EGRESS); \ ++ } \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_SK_PROG(sk, type) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) { \ ++ __ret = __cgroup_bpf_run_filter_sk(sk, type); \ ++ } \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) \ ++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE) ++ ++#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) \ ++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND) ++ ++#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) \ ++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND) ++ ++#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ ++ NULL); \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) { \ ++ lock_sock(sk); \ ++ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ ++ t_ctx); \ ++ release_sock(sk); \ ++ } \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_BIND) ++ ++#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_BIND) ++ ++#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \ ++ sk->sk_prot->pre_connect) ++ ++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT) ++ ++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT) ++ ++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL) ++ ++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL) ++ ++#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx) ++ ++#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx) ++ ++#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL) ++ ++#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) \ ++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL) ++ ++#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled && (sock_ops)->sk) { \ ++ typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ ++ if (__sk && sk_fullsock(__sk)) \ ++ __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ ++ sock_ops, \ ++ BPF_CGROUP_SOCK_OPS); \ ++ } \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ ++ access, \ ++ BPF_CGROUP_DEVICE); \ ++ \ ++ __ret; \ ++}) ++ ++ ++#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos, nbuf) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ ++ buf, count, pos, nbuf, \ ++ BPF_CGROUP_SYSCTL); \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ ++ kernel_optval) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ ++ optname, optval, \ ++ optlen, \ ++ kernel_optval); \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \ ++({ \ ++ int __ret = 0; \ ++ if (cgroup_bpf_enabled) \ ++ get_user(__ret, optlen); \ ++ __ret; \ ++}) ++ ++#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen, \ ++ max_optlen, retval) \ ++({ \ ++ int __ret = retval; \ ++ if (cgroup_bpf_enabled) \ ++ __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \ ++ optname, optval, \ ++ optlen, max_optlen, \ ++ retval); \ ++ __ret; \ ++}) ++ ++int cgroup_bpf_prog_attach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype, struct bpf_prog *prog); ++int cgroup_bpf_prog_detach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype); ++int cgroup_bpf_prog_query(const union bpf_attr *attr, ++ union bpf_attr __user *uattr); ++#else ++ ++struct bpf_prog; ++struct cgroup_bpf {}; ++static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; } ++static inline void cgroup_bpf_offline(struct cgroup *cgrp) {} ++ ++static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype, ++ struct bpf_prog *prog) ++{ ++ return -EINVAL; ++} ++ ++static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype) ++{ ++ return -EINVAL; ++} ++ ++static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ return -EINVAL; ++} ++ ++static inline void bpf_cgroup_storage_set( ++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {} ++static inline int bpf_cgroup_storage_assign(struct bpf_prog *prog, ++ struct bpf_map *map) { return 0; } ++static inline void bpf_cgroup_storage_release(struct bpf_prog *prog, ++ struct bpf_map *map) {} ++static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( ++ struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; } ++static inline void bpf_cgroup_storage_free( ++ struct bpf_cgroup_storage *storage) {} ++static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, ++ void *value) { ++ return 0; ++} ++static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, ++ void *key, void *value, u64 flags) { ++ return 0; ++} ++ ++#define cgroup_bpf_enabled (0) ++#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0) ++#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos,nbuf) ({ 0; }) ++#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; }) ++#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ ++ optlen, max_optlen, retval) ({ retval; }) ++#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ ++ kernel_optval) ({ 0; }) ++ ++#define for_each_cgroup_storage_type(stype) for (; false; ) ++ ++#endif /* CONFIG_CGROUP_BPF */ ++ ++#endif /* _BPF_CGROUP_H */ +--- a/include/linux/bpf.h ++++ b/include/linux/bpf.h +@@ -1,55 +1,183 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. + */ + #ifndef _LINUX_BPF_H + #define _LINUX_BPF_H 1 + + #include ++ + #include + #include ++#include ++#include ++#include ++#include ++#include ++#include + ++struct bpf_verifier_env; ++struct perf_event; ++struct bpf_prog; + struct bpf_map; ++struct sock; ++struct seq_file; ++struct btf; ++struct btf_type; ++ ++extern struct idr btf_idr; ++extern spinlock_t btf_idr_lock; + + /* map is generic key/value storage optionally accesible by eBPF programs */ + struct bpf_map_ops { + /* funcs callable from userspace (via syscall) */ ++ int (*map_alloc_check)(union bpf_attr *attr); + struct bpf_map *(*map_alloc)(union bpf_attr *attr); +- void (*map_free)(struct bpf_map *); ++ void (*map_release)(struct bpf_map *map, struct file *map_file); ++ void (*map_free)(struct bpf_map *map); + int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); ++ void (*map_release_uref)(struct bpf_map *map); ++ void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); + + /* funcs callable from userspace and from eBPF programs */ + void *(*map_lookup_elem)(struct bpf_map *map, void *key); + int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); + int (*map_delete_elem)(struct bpf_map *map, void *key); ++ int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); ++ int (*map_pop_elem)(struct bpf_map *map, void *value); ++ int (*map_peek_elem)(struct bpf_map *map, void *value); + + /* funcs called by prog_array and perf_event_array map */ +- void *(*map_fd_get_ptr) (struct bpf_map *map, int fd); +- void (*map_fd_put_ptr) (void *ptr); ++ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, ++ int fd); ++ void (*map_fd_put_ptr)(void *ptr); ++ u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); ++ u32 (*map_fd_sys_lookup_elem)(void *ptr); ++ void (*map_seq_show_elem)(struct bpf_map *map, void *key, ++ struct seq_file *m); ++ int (*map_check_btf)(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type); ++ ++ /* Direct value access helpers. */ ++ int (*map_direct_value_addr)(const struct bpf_map *map, ++ u64 *imm, u32 off); ++ int (*map_direct_value_meta)(const struct bpf_map *map, ++ u64 imm, u32 *off); ++}; ++ ++struct bpf_map_memory { ++ u32 pages; ++ struct user_struct *user; + }; + + struct bpf_map { +- atomic_t refcnt; ++ /* The first two cachelines with read-mostly members of which some ++ * are also accessed in fast-path (e.g. ops, max_entries). ++ */ ++ const struct bpf_map_ops *ops ____cacheline_aligned; ++ struct bpf_map *inner_map_meta; ++#ifdef CONFIG_SECURITY ++ void *security; ++#endif + enum bpf_map_type map_type; + u32 key_size; + u32 value_size; + u32 max_entries; +- u32 pages; ++ u32 map_flags; ++ int spin_lock_off; /* >=0 valid offset, <0 error */ ++ u32 id; ++ int numa_node; ++ u32 btf_key_type_id; ++ u32 btf_value_type_id; ++ struct btf *btf; ++ struct bpf_map_memory memory; + bool unpriv_array; +- struct user_struct *user; +- const struct bpf_map_ops *ops; +- struct work_struct work; ++ bool frozen; /* write-once */ ++ /* 48 bytes hole */ ++ ++ /* The 3rd and 4th cacheline with misc members to avoid false sharing ++ * particularly with refcounting. ++ */ ++ atomic_t refcnt ____cacheline_aligned; + atomic_t usercnt; ++ struct work_struct work; ++ char name[BPF_OBJ_NAME_LEN]; + }; + +-struct bpf_map_type_list { +- struct list_head list_node; +- const struct bpf_map_ops *ops; +- enum bpf_map_type type; ++static inline bool map_value_has_spin_lock(const struct bpf_map *map) ++{ ++ return map->spin_lock_off >= 0; ++} ++ ++static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) ++{ ++ if (likely(!map_value_has_spin_lock(map))) ++ return; ++ *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = ++ (struct bpf_spin_lock){}; ++} ++ ++/* copy everything but bpf_spin_lock */ ++static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) ++{ ++ if (unlikely(map_value_has_spin_lock(map))) { ++ u32 off = map->spin_lock_off; ++ ++ memcpy(dst, src, off); ++ memcpy(dst + off + sizeof(struct bpf_spin_lock), ++ src + off + sizeof(struct bpf_spin_lock), ++ map->value_size - off - sizeof(struct bpf_spin_lock)); ++ } else { ++ memcpy(dst, src, map->value_size); ++ } ++} ++void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, ++ bool lock_src); ++ ++struct bpf_offload_dev; ++struct bpf_offloaded_map; ++ ++struct bpf_map_dev_ops { ++ int (*map_get_next_key)(struct bpf_offloaded_map *map, ++ void *key, void *next_key); ++ int (*map_lookup_elem)(struct bpf_offloaded_map *map, ++ void *key, void *value); ++ int (*map_update_elem)(struct bpf_offloaded_map *map, ++ void *key, void *value, u64 flags); ++ int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); + }; + ++struct bpf_offloaded_map { ++ struct bpf_map map; ++ struct net_device *netdev; ++ const struct bpf_map_dev_ops *dev_ops; ++ void *dev_priv; ++ struct list_head offloads; ++}; ++ ++static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) ++{ ++ return container_of(map, struct bpf_offloaded_map, map); ++} ++ ++static inline bool bpf_map_offload_neutral(const struct bpf_map *map) ++{ ++ return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; ++} ++ ++static inline bool bpf_map_support_seq_show(const struct bpf_map *map) ++{ ++ return map->btf && map->ops->map_seq_show_elem; ++} ++ ++int map_check_no_btf(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type); ++ ++extern const struct bpf_map_ops bpf_map_offload_ops; ++ + /* function argument constraints */ + enum bpf_arg_type { + ARG_DONTCARE = 0, /* unused argument in helper function */ +@@ -60,22 +188,40 @@ enum bpf_arg_type { + ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ + ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ + ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ ++ ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ ++ ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ + + /* the following constraints used to prototype bpf_memcmp() and other + * functions that access data on eBPF program stack + */ +- ARG_PTR_TO_STACK, /* any pointer to eBPF program stack */ +- ARG_CONST_STACK_SIZE, /* number of bytes accessed from stack */ ++ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ++ ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ ++ ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, ++ * helper function must fill all bytes or clear ++ * them in error case. ++ */ ++ ++ ARG_CONST_SIZE, /* number of bytes accessed from memory */ ++ ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + + ARG_PTR_TO_CTX, /* pointer to context */ + ARG_ANYTHING, /* any (initialized) argument is ok */ ++ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ++ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ++ ARG_PTR_TO_INT, /* pointer to int */ ++ ARG_PTR_TO_LONG, /* pointer to long */ ++ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ + }; + + /* type of values returned from helper functions */ + enum bpf_return_type { + RET_INTEGER, /* function returns integer */ + RET_VOID, /* function doesn't return anything */ ++ RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ + RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ ++ RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ ++ RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ ++ RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ + }; + + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs +@@ -85,6 +231,7 @@ enum bpf_return_type { + struct bpf_func_proto { + u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); + bool gpl_only; ++ bool pkt_access; + enum bpf_return_type ret_type; + enum bpf_arg_type arg1_type; + enum bpf_arg_type arg2_type; +@@ -104,35 +251,172 @@ enum bpf_access_type { + BPF_WRITE = 2 + }; + +-struct bpf_prog; ++/* types of values stored in eBPF registers */ ++/* Pointer types represent: ++ * pointer ++ * pointer + imm ++ * pointer + (u16) var ++ * pointer + (u16) var + imm ++ * if (range > 0) then [ptr, ptr + range - off) is safe to access ++ * if (id > 0) means that some 'var' was added ++ * if (off > 0) means that 'imm' was added ++ */ ++enum bpf_reg_type { ++ NOT_INIT = 0, /* nothing was written into register */ ++ SCALAR_VALUE, /* reg doesn't contain a valid pointer */ ++ PTR_TO_CTX, /* reg points to bpf_context */ ++ CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ ++ PTR_TO_MAP_VALUE, /* reg points to map element value */ ++ PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ ++ PTR_TO_STACK, /* reg == frame_pointer + offset */ ++ PTR_TO_PACKET_META, /* skb->data - meta_len */ ++ PTR_TO_PACKET, /* reg points to skb->data */ ++ PTR_TO_PACKET_END, /* skb->data + headlen */ ++ PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ ++ PTR_TO_SOCKET, /* reg points to struct bpf_sock */ ++ PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ ++ PTR_TO_SOCK_COMMON, /* reg points to sock_common */ ++ PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ ++ PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ ++ PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ ++ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ ++ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ ++}; ++ ++/* The information passed from prog-specific *_is_valid_access ++ * back to the verifier. ++ */ ++struct bpf_insn_access_aux { ++ enum bpf_reg_type reg_type; ++ int ctx_field_size; ++}; ++ ++static inline void ++bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) ++{ ++ aux->ctx_field_size = size; ++} ++ ++struct bpf_prog_ops { ++ int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, ++ union bpf_attr __user *uattr); ++}; + + struct bpf_verifier_ops { + /* return eBPF function prototype for verification */ +- const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id); ++ const struct bpf_func_proto * ++ (*get_func_proto)(enum bpf_func_id func_id, ++ const struct bpf_prog *prog); + + /* return true if 'size' wide access at offset 'off' within bpf_context + * with 'type' (read or write) is allowed + */ +- bool (*is_valid_access)(int off, int size, enum bpf_access_type type); ++ bool (*is_valid_access)(int off, int size, enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info); ++ int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, ++ const struct bpf_prog *prog); ++ int (*gen_ld_abs)(const struct bpf_insn *orig, ++ struct bpf_insn *insn_buf); ++ u32 (*convert_ctx_access)(enum bpf_access_type type, ++ const struct bpf_insn *src, ++ struct bpf_insn *dst, ++ struct bpf_prog *prog, u32 *target_size); ++}; + +- u32 (*convert_ctx_access)(enum bpf_access_type type, int dst_reg, +- int src_reg, int ctx_off, +- struct bpf_insn *insn, struct bpf_prog *prog); ++struct bpf_prog_offload_ops { ++ /* verifier basic callbacks */ ++ int (*insn_hook)(struct bpf_verifier_env *env, ++ int insn_idx, int prev_insn_idx); ++ int (*finalize)(struct bpf_verifier_env *env); ++ /* verifier optimization callbacks (called after .finalize) */ ++ int (*replace_insn)(struct bpf_verifier_env *env, u32 off, ++ struct bpf_insn *insn); ++ int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); ++ /* program management callbacks */ ++ int (*prepare)(struct bpf_prog *prog); ++ int (*translate)(struct bpf_prog *prog); ++ void (*destroy)(struct bpf_prog *prog); + }; + +-struct bpf_prog_type_list { +- struct list_head list_node; +- const struct bpf_verifier_ops *ops; +- enum bpf_prog_type type; ++struct bpf_prog_offload { ++ struct bpf_prog *prog; ++ struct net_device *netdev; ++ struct bpf_offload_dev *offdev; ++ void *dev_priv; ++ struct list_head offloads; ++ bool dev_state; ++ bool opt_failed; ++ void *jited_image; ++ u32 jited_len; ++}; ++ ++enum bpf_cgroup_storage_type { ++ BPF_CGROUP_STORAGE_SHARED, ++ BPF_CGROUP_STORAGE_PERCPU, ++ __BPF_CGROUP_STORAGE_MAX ++}; ++ ++#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX ++ ++struct bpf_prog_stats { ++ u64 cnt; ++ u64 nsecs; ++ struct u64_stats_sync syncp; + }; + + struct bpf_prog_aux { + atomic_t refcnt; + u32 used_map_cnt; +- const struct bpf_verifier_ops *ops; ++ u32 max_ctx_offset; ++ u32 max_pkt_offset; ++ u32 max_tp_access; ++ u32 stack_depth; ++ u32 id; ++ u32 func_cnt; /* used by non-func prog as the number of func progs */ ++ u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ ++ bool verifier_zext; /* Zero extensions has been inserted by verifier. */ ++ bool offload_requested; ++ struct bpf_prog **func; ++ void *jit_data; /* JIT specific data. arch dependent */ ++ struct latch_tree_node ksym_tnode; ++ struct list_head ksym_lnode; ++ const struct bpf_prog_ops *ops; + struct bpf_map **used_maps; + struct bpf_prog *prog; + struct user_struct *user; ++ u64 load_time; /* ns since boottime */ ++ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; ++ char name[BPF_OBJ_NAME_LEN]; ++#ifdef CONFIG_SECURITY ++ void *security; ++#endif ++ struct bpf_prog_offload *offload; ++ struct btf *btf; ++ struct bpf_func_info *func_info; ++ /* bpf_line_info loaded from userspace. linfo->insn_off ++ * has the xlated insn offset. ++ * Both the main and sub prog share the same linfo. ++ * The subprog can access its first linfo by ++ * using the linfo_idx. ++ */ ++ struct bpf_line_info *linfo; ++ /* jited_linfo is the jited addr of the linfo. It has a ++ * one to one mapping to linfo: ++ * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. ++ * Both the main and sub prog share the same jited_linfo. ++ * The subprog can access its first jited_linfo by ++ * using the linfo_idx. ++ */ ++ void **jited_linfo; ++ u32 func_info_cnt; ++ u32 nr_linfo; ++ /* subprog can use linfo_idx to access its first linfo and ++ * jited_linfo. ++ * main prog always has linfo_idx == 0 ++ */ ++ u32 linfo_idx; ++ struct bpf_prog_stats __percpu *stats; + union { + struct work_struct work; + struct rcu_head rcu; +@@ -153,76 +437,688 @@ struct bpf_array { + union { + char value[0] __aligned(8); + void *ptrs[0] __aligned(8); ++ void __percpu *pptrs[0] __aligned(8); + }; + }; ++ ++#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ + #define MAX_TAIL_CALL_CNT 32 + +-u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5); +-void bpf_fd_array_map_clear(struct bpf_map *map); ++#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ ++ BPF_F_RDONLY_PROG | \ ++ BPF_F_WRONLY | \ ++ BPF_F_WRONLY_PROG) ++ ++#define BPF_MAP_CAN_READ BIT(0) ++#define BPF_MAP_CAN_WRITE BIT(1) ++ ++static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) ++{ ++ u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); ++ ++ /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is ++ * not possible. ++ */ ++ if (access_flags & BPF_F_RDONLY_PROG) ++ return BPF_MAP_CAN_READ; ++ else if (access_flags & BPF_F_WRONLY_PROG) ++ return BPF_MAP_CAN_WRITE; ++ else ++ return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; ++} ++ ++static inline bool bpf_map_flags_access_ok(u32 access_flags) ++{ ++ return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != ++ (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); ++} ++ ++struct bpf_event_entry { ++ struct perf_event *event; ++ struct file *perf_file; ++ struct file *map_file; ++ struct rcu_head rcu; ++}; ++ + bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); ++int bpf_prog_calc_tag(struct bpf_prog *fp); ++ + const struct bpf_func_proto *bpf_get_trace_printk_proto(void); + ++typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, ++ unsigned long off, unsigned long len); ++typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, ++ const struct bpf_insn *src, ++ struct bpf_insn *dst, ++ struct bpf_prog *prog, ++ u32 *target_size); ++ ++u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, ++ void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); ++ ++/* an array of programs to be executed under rcu_lock. ++ * ++ * Typical usage: ++ * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); ++ * ++ * the structure returned by bpf_prog_array_alloc() should be populated ++ * with program pointers and the last pointer must be NULL. ++ * The user has to keep refcnt on the program and make sure the program ++ * is removed from the array before bpf_prog_put(). ++ * The 'struct bpf_prog_array *' should only be replaced with xchg() ++ * since other cpus are walking the array of pointers in parallel. ++ */ ++struct bpf_prog_array_item { ++ struct bpf_prog *prog; ++ struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; ++}; ++ ++struct bpf_prog_array { ++ struct rcu_head rcu; ++ struct bpf_prog_array_item items[0]; ++}; ++ ++struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); ++void bpf_prog_array_free(struct bpf_prog_array *progs); ++int bpf_prog_array_length(struct bpf_prog_array *progs); ++bool bpf_prog_array_is_empty(struct bpf_prog_array *array); ++int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, ++ __u32 __user *prog_ids, u32 cnt); ++ ++void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, ++ struct bpf_prog *old_prog); ++int bpf_prog_array_copy_info(struct bpf_prog_array *array, ++ u32 *prog_ids, u32 request_cnt, ++ u32 *prog_cnt); ++int bpf_prog_array_copy(struct bpf_prog_array *old_array, ++ struct bpf_prog *exclude_prog, ++ struct bpf_prog *include_prog, ++ struct bpf_prog_array **new_array); ++ ++#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null, set_cg_storage) \ ++ ({ \ ++ struct bpf_prog_array_item *_item; \ ++ struct bpf_prog *_prog; \ ++ struct bpf_prog_array *_array; \ ++ u32 _ret = 1; \ ++ preempt_disable(); \ ++ rcu_read_lock(); \ ++ _array = rcu_dereference(array); \ ++ if (unlikely(check_non_null && !_array))\ ++ goto _out; \ ++ _item = &_array->items[0]; \ ++ while ((_prog = READ_ONCE(_item->prog))) { \ ++ if (set_cg_storage) \ ++ bpf_cgroup_storage_set(_item->cgroup_storage); \ ++ _ret &= func(_prog, ctx); \ ++ _item++; \ ++ } \ ++_out: \ ++ rcu_read_unlock(); \ ++ preempt_enable(); \ ++ _ret; \ ++ }) ++ ++/* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs ++ * so BPF programs can request cwr for TCP packets. ++ * ++ * Current cgroup skb programs can only return 0 or 1 (0 to drop the ++ * packet. This macro changes the behavior so the low order bit ++ * indicates whether the packet should be dropped (0) or not (1) ++ * and the next bit is a congestion notification bit. This could be ++ * used by TCP to call tcp_enter_cwr() ++ * ++ * Hence, new allowed return values of CGROUP EGRESS BPF programs are: ++ * 0: drop packet ++ * 1: keep packet ++ * 2: drop packet and cn ++ * 3: keep packet and cn ++ * ++ * This macro then converts it to one of the NET_XMIT or an error ++ * code that is then interpreted as drop packet (and no cn): ++ * 0: NET_XMIT_SUCCESS skb should be transmitted ++ * 1: NET_XMIT_DROP skb should be dropped and cn ++ * 2: NET_XMIT_CN skb should be transmitted and cn ++ * 3: -EPERM skb should be dropped ++ */ ++#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ ++ ({ \ ++ struct bpf_prog_array_item *_item; \ ++ struct bpf_prog *_prog; \ ++ struct bpf_prog_array *_array; \ ++ u32 ret; \ ++ u32 _ret = 1; \ ++ u32 _cn = 0; \ ++ preempt_disable(); \ ++ rcu_read_lock(); \ ++ _array = rcu_dereference(array); \ ++ _item = &_array->items[0]; \ ++ while ((_prog = READ_ONCE(_item->prog))) { \ ++ bpf_cgroup_storage_set(_item->cgroup_storage); \ ++ ret = func(_prog, ctx); \ ++ _ret &= (ret & 1); \ ++ _cn |= (ret & 2); \ ++ _item++; \ ++ } \ ++ rcu_read_unlock(); \ ++ preempt_enable(); \ ++ if (_ret) \ ++ _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ ++ else \ ++ _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ ++ _ret; \ ++ }) ++ ++#define BPF_PROG_RUN_ARRAY(array, ctx, func) \ ++ __BPF_PROG_RUN_ARRAY(array, ctx, func, false, true) ++ ++#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ ++ __BPF_PROG_RUN_ARRAY(array, ctx, func, true, false) ++ + #ifdef CONFIG_BPF_SYSCALL +-void bpf_register_prog_type(struct bpf_prog_type_list *tl); +-void bpf_register_map_type(struct bpf_map_type_list *tl); ++DECLARE_PER_CPU(int, bpf_prog_active); ++ ++extern const struct file_operations bpf_map_fops; ++extern const struct file_operations bpf_prog_fops; ++ ++#define BPF_PROG_TYPE(_id, _name) \ ++ extern const struct bpf_prog_ops _name ## _prog_ops; \ ++ extern const struct bpf_verifier_ops _name ## _verifier_ops; ++#define BPF_MAP_TYPE(_id, _ops) \ ++ extern const struct bpf_map_ops _ops; ++#include ++#undef BPF_PROG_TYPE ++#undef BPF_MAP_TYPE ++ ++extern const struct bpf_prog_ops bpf_offload_prog_ops; ++extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; ++extern const struct bpf_verifier_ops xdp_analyzer_ops; + + struct bpf_prog *bpf_prog_get(u32 ufd); +-struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog); ++struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, ++ bool attach_drv); ++struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i); ++void bpf_prog_sub(struct bpf_prog *prog, int i); ++struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog); ++struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); + void bpf_prog_put(struct bpf_prog *prog); +-void bpf_prog_put_rcu(struct bpf_prog *prog); ++int __bpf_prog_charge(struct user_struct *user, u32 pages); ++void __bpf_prog_uncharge(struct user_struct *user, u32 pages); ++ ++void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); ++void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); + + struct bpf_map *bpf_map_get_with_uref(u32 ufd); + struct bpf_map *__bpf_map_get(struct fd f); +-struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref); ++struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); ++struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map, ++ bool uref); + void bpf_map_put_with_uref(struct bpf_map *map); + void bpf_map_put(struct bpf_map *map); ++int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); ++void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); ++int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size); ++void bpf_map_charge_finish(struct bpf_map_memory *mem); ++void bpf_map_charge_move(struct bpf_map_memory *dst, ++ struct bpf_map_memory *src); ++void *bpf_map_area_alloc(u64 size, int numa_node); ++void bpf_map_area_free(void *base); ++void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); + + extern int sysctl_unprivileged_bpf_disabled; + +-int bpf_map_new_fd(struct bpf_map *map); ++int bpf_map_new_fd(struct bpf_map *map, int flags); + int bpf_prog_new_fd(struct bpf_prog *prog); + + int bpf_obj_pin_user(u32 ufd, const char __user *pathname); +-int bpf_obj_get_user(const char __user *pathname); ++int bpf_obj_get_user(const char __user *pathname, int flags); ++ ++int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); ++int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); ++int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, ++ u64 flags); ++int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, ++ u64 flags); ++ ++int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); ++ ++int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, ++ void *key, void *value, u64 map_flags); ++int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); ++int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, ++ void *key, void *value, u64 map_flags); ++int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); ++ ++int bpf_get_file_flag(int flags); ++int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, ++ size_t actual_size); ++ ++/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and ++ * forced to use 'long' read/writes to try to atomically copy long counters. ++ * Best-effort only. No barriers here, since it _will_ race with concurrent ++ * updates from BPF programs. Called from bpf syscall and mostly used with ++ * size 8 or 16 bytes, so ask compiler to inline it. ++ */ ++static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) ++{ ++ const long *lsrc = src; ++ long *ldst = dst; ++ ++ size /= sizeof(long); ++ while (size--) ++ *ldst++ = *lsrc++; ++} + + /* verify correctness of eBPF program */ +-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr); +-#else +-static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl) ++int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, ++ union bpf_attr __user *uattr); ++ ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); ++#endif ++ ++/* Map specifics */ ++struct xdp_buff; ++struct sk_buff; ++ ++struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); ++struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); ++void __dev_map_flush(struct bpf_map *map); ++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, ++ struct net_device *dev_rx); ++int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, ++ struct bpf_prog *xdp_prog); ++ ++struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); ++void __cpu_map_flush(struct bpf_map *map); ++int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, ++ struct net_device *dev_rx); ++ ++/* Return map's numa specified by userspace */ ++static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) + { ++ return (attr->map_flags & BPF_F_NUMA_NODE) ? ++ attr->numa_node : NUMA_NO_NODE; + } + ++struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); ++int array_map_alloc_check(union bpf_attr *attr); ++ ++int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, ++ union bpf_attr __user *uattr); ++int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, ++ union bpf_attr __user *uattr); ++int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, ++ const union bpf_attr *kattr, ++ union bpf_attr __user *uattr); ++#else /* !CONFIG_BPF_SYSCALL */ + static inline struct bpf_prog *bpf_prog_get(u32 ufd) + { + return ERR_PTR(-EOPNOTSUPP); + } + ++static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, ++ enum bpf_prog_type type, ++ bool attach_drv) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, ++ int i) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++static inline void bpf_prog_sub(struct bpf_prog *prog, int i) ++{ ++} ++ + static inline void bpf_prog_put(struct bpf_prog *prog) + { + } + +-static inline void bpf_prog_put_rcu(struct bpf_prog *prog) ++static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++static inline struct bpf_prog *__must_check ++bpf_prog_inc_not_zero(struct bpf_prog *prog) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) ++{ ++ return 0; ++} ++ ++static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) ++{ ++} ++ ++static inline int bpf_obj_get_user(const char __user *pathname, int flags) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, ++ u32 key) ++{ ++ return NULL; ++} ++ ++static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, ++ u32 key) ++{ ++ return NULL; ++} ++ ++static inline void __dev_map_flush(struct bpf_map *map) ++{ ++} ++ ++struct xdp_buff; ++struct bpf_dtab_netdev; ++ ++static inline ++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, ++ struct net_device *dev_rx) ++{ ++ return 0; ++} ++ ++struct sk_buff; ++ ++static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, ++ struct sk_buff *skb, ++ struct bpf_prog *xdp_prog) ++{ ++ return 0; ++} ++ ++static inline ++struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) ++{ ++ return NULL; ++} ++ ++static inline void __cpu_map_flush(struct bpf_map *map) ++{ ++} ++ ++static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, ++ struct xdp_buff *xdp, ++ struct net_device *dev_rx) ++{ ++ return 0; ++} ++ ++static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, ++ enum bpf_prog_type type) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, ++ const union bpf_attr *kattr, ++ union bpf_attr __user *uattr) ++{ ++ return -ENOTSUPP; ++} ++ ++static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, ++ const union bpf_attr *kattr, ++ union bpf_attr __user *uattr) ++{ ++ return -ENOTSUPP; ++} ++ ++static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, ++ const union bpf_attr *kattr, ++ union bpf_attr __user *uattr) ++{ ++ return -ENOTSUPP; ++} ++#endif /* CONFIG_BPF_SYSCALL */ ++ ++static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, ++ enum bpf_prog_type type) ++{ ++ return bpf_prog_get_type_dev(ufd, type, false); ++} ++ ++bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); ++ ++#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) ++ ++static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) ++{ ++ return aux->offload_requested; ++} ++ ++static inline bool bpf_map_is_dev_bound(struct bpf_map *map) ++{ ++ return false; ++} ++ ++#else ++static inline int bpf_prog_offload_init(struct bpf_prog *prog, ++ union bpf_attr *attr) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) ++{ ++ return false; ++} ++ ++static inline bool bpf_map_is_dev_bound(struct bpf_map *map) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ ++ ++#if defined(CONFIG_BPF_STREAM_PARSER) ++int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, ++ struct bpf_prog *old, u32 which); ++int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); ++int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); ++#else ++static inline int sock_map_prog_update(struct bpf_map *map, ++ struct bpf_prog *prog, ++ struct bpf_prog *old, u32 which) + { ++ return -EOPNOTSUPP; ++} ++ ++static inline int sock_map_get_from_fd(const union bpf_attr *attr, ++ struct bpf_prog *prog) ++{ ++ return -EINVAL; ++} ++ ++static inline int sock_map_prog_detach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype) ++{ ++ return -EOPNOTSUPP; ++} ++#endif ++ ++#if defined(CONFIG_XDP_SOCKETS) ++struct xdp_sock; ++struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key); ++int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, ++ struct xdp_sock *xs); ++void __xsk_map_flush(struct bpf_map *map); ++#else ++struct xdp_sock; ++static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, ++ u32 key) ++{ ++ return NULL; ++} ++ ++static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, ++ struct xdp_sock *xs) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline void __xsk_map_flush(struct bpf_map *map) ++{ ++} ++#endif ++ ++#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) ++void bpf_sk_reuseport_detach(struct sock *sk); ++int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, ++ void *value); ++int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 map_flags); ++#else ++static inline void bpf_sk_reuseport_detach(struct sock *sk) ++{ ++} ++ ++#ifdef CONFIG_BPF_SYSCALL ++static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, ++ void *key, void *value) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, ++ void *key, void *value, ++ u64 map_flags) ++{ ++ return -EOPNOTSUPP; + } + #endif /* CONFIG_BPF_SYSCALL */ ++#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ + + /* verifier prototypes for helper functions called from eBPF programs */ + extern const struct bpf_func_proto bpf_map_lookup_elem_proto; + extern const struct bpf_func_proto bpf_map_update_elem_proto; + extern const struct bpf_func_proto bpf_map_delete_elem_proto; ++extern const struct bpf_func_proto bpf_map_push_elem_proto; ++extern const struct bpf_func_proto bpf_map_pop_elem_proto; ++extern const struct bpf_func_proto bpf_map_peek_elem_proto; + + extern const struct bpf_func_proto bpf_get_prandom_u32_proto; + extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; ++extern const struct bpf_func_proto bpf_get_numa_node_id_proto; + extern const struct bpf_func_proto bpf_tail_call_proto; + extern const struct bpf_func_proto bpf_ktime_get_ns_proto; + extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; + extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; + extern const struct bpf_func_proto bpf_get_current_comm_proto; +-extern const struct bpf_func_proto bpf_skb_vlan_push_proto; +-extern const struct bpf_func_proto bpf_skb_vlan_pop_proto; ++extern const struct bpf_func_proto bpf_get_stackid_proto; ++extern const struct bpf_func_proto bpf_get_stack_proto; ++extern const struct bpf_func_proto bpf_sock_map_update_proto; ++extern const struct bpf_func_proto bpf_sock_hash_update_proto; ++extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; ++extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; ++extern const struct bpf_func_proto bpf_msg_redirect_map_proto; ++extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; ++extern const struct bpf_func_proto bpf_sk_redirect_map_proto; ++extern const struct bpf_func_proto bpf_spin_lock_proto; ++extern const struct bpf_func_proto bpf_spin_unlock_proto; ++extern const struct bpf_func_proto bpf_get_local_storage_proto; ++extern const struct bpf_func_proto bpf_strtol_proto; ++extern const struct bpf_func_proto bpf_strtoul_proto; ++extern const struct bpf_func_proto bpf_tcp_sock_proto; + + /* Shared helpers among cBPF and eBPF. */ + void bpf_user_rnd_init_once(void); + u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); + ++#if defined(CONFIG_NET) ++bool bpf_sock_common_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info); ++bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, ++ struct bpf_insn_access_aux *info); ++u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size); ++#else ++static inline bool bpf_sock_common_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++static inline bool bpf_sock_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++{ ++ return 0; ++} ++#endif ++ ++#ifdef CONFIG_INET ++bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, ++ struct bpf_insn_access_aux *info); ++ ++u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size); ++ ++bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, ++ struct bpf_insn_access_aux *info); ++ ++u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size); ++#else ++static inline bool bpf_tcp_sock_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++{ ++ return 0; ++} ++static inline bool bpf_xdp_sock_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++{ ++ return 0; ++} ++#endif /* CONFIG_INET */ ++ + #endif /* _LINUX_BPF_H */ +--- /dev/null ++++ b/include/linux/bpf_trace.h +@@ -0,0 +1,7 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef __LINUX_BPF_TRACE_H__ ++#define __LINUX_BPF_TRACE_H__ ++ ++#include ++ ++#endif /* __LINUX_BPF_TRACE_H__ */ +--- /dev/null ++++ b/include/linux/bpf_types.h +@@ -0,0 +1,44 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* internal file - do not include directly */ ++ ++#ifdef CONFIG_NET ++BPF_PROG_TYPE(BPF_PROG_TYPE_SOCKET_FILTER, sk_filter) ++BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_CLS, tc_cls_act) ++BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_ACT, tc_cls_act) ++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_IN, lwt_in) ++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_OUT, lwt_out) ++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_XMIT, lwt_xmit) ++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_SEG6LOCAL, lwt_seg6local) ++BPF_PROG_TYPE(BPF_PROG_TYPE_SOCK_OPS, sock_ops) ++BPF_PROG_TYPE(BPF_PROG_TYPE_SK_SKB, sk_skb) ++BPF_PROG_TYPE(BPF_PROG_TYPE_SK_MSG, sk_msg) ++BPF_PROG_TYPE(BPF_PROG_TYPE_FLOW_DISSECTOR, flow_dissector) ++#endif ++#ifdef CONFIG_BPF_EVENTS ++BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe) ++BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint) ++BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event) ++BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT, raw_tracepoint) ++BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, raw_tracepoint_writable) ++#endif ++ ++BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_PROG_ARRAY, prog_array_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_PERF_EVENT_ARRAY, perf_event_array_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_HASH, htab_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_HASH, htab_percpu_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_HASH, htab_lru_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_PERCPU_HASH, htab_lru_percpu_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_LPM_TRIE, trie_map_ops) ++#ifdef CONFIG_PERF_EVENTS ++BPF_MAP_TYPE(BPF_MAP_TYPE_STACK_TRACE, stack_trace_map_ops) ++#endif ++BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_of_maps_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops) ++#ifdef CONFIG_NET ++BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP_HASH, dev_map_hash_ops) ++#endif ++BPF_MAP_TYPE(BPF_MAP_TYPE_QUEUE, queue_map_ops) ++BPF_MAP_TYPE(BPF_MAP_TYPE_STACK, stack_map_ops) +--- /dev/null ++++ b/include/linux/bpf_verifier.h +@@ -0,0 +1,425 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com ++ */ ++#ifndef _LINUX_BPF_VERIFIER_H ++#define _LINUX_BPF_VERIFIER_H 1 ++ ++#include /* for enum bpf_reg_type */ ++#include /* for MAX_BPF_STACK */ ++#include ++ ++/* Maximum variable offset umax_value permitted when resolving memory accesses. ++ * In practice this is far bigger than any realistic pointer offset; this limit ++ * ensures that umax_value + (int)off + (int)size cannot overflow a u64. ++ */ ++#define BPF_MAX_VAR_OFF (1 << 29) ++/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures ++ * that converting umax_value to int cannot overflow. ++ */ ++#define BPF_MAX_VAR_SIZ (1 << 29) ++ ++/* Liveness marks, used for registers and spilled-regs (in stack slots). ++ * Read marks propagate upwards until they find a write mark; they record that ++ * "one of this state's descendants read this reg" (and therefore the reg is ++ * relevant for states_equal() checks). ++ * Write marks collect downwards and do not propagate; they record that "the ++ * straight-line code that reached this state (from its parent) wrote this reg" ++ * (and therefore that reads propagated from this state or its descendants ++ * should not propagate to its parent). ++ * A state with a write mark can receive read marks; it just won't propagate ++ * them to its parent, since the write mark is a property, not of the state, ++ * but of the link between it and its parent. See mark_reg_read() and ++ * mark_stack_slot_read() in kernel/bpf/verifier.c. ++ */ ++enum bpf_reg_liveness { ++ REG_LIVE_NONE = 0, /* reg hasn't been read or written this branch */ ++ REG_LIVE_READ32 = 0x1, /* reg was read, so we're sensitive to initial value */ ++ REG_LIVE_READ64 = 0x2, /* likewise, but full 64-bit content matters */ ++ REG_LIVE_READ = REG_LIVE_READ32 | REG_LIVE_READ64, ++ REG_LIVE_WRITTEN = 0x4, /* reg was written first, screening off later reads */ ++ REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */ ++}; ++ ++struct bpf_reg_state { ++ /* Ordering of fields matters. See states_equal() */ ++ enum bpf_reg_type type; ++ union { ++ /* valid when type == PTR_TO_PACKET */ ++ u16 range; ++ ++ /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | ++ * PTR_TO_MAP_VALUE_OR_NULL ++ */ ++ struct bpf_map *map_ptr; ++ ++ /* Max size from any of the above. */ ++ unsigned long raw; ++ }; ++ /* Fixed part of pointer offset, pointer types only */ ++ s32 off; ++ /* For PTR_TO_PACKET, used to find other pointers with the same variable ++ * offset, so they can share range knowledge. ++ * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we ++ * came from, when one is tested for != NULL. ++ * For PTR_TO_SOCKET this is used to share which pointers retain the ++ * same reference to the socket, to determine proper reference freeing. ++ */ ++ u32 id; ++ /* PTR_TO_SOCKET and PTR_TO_TCP_SOCK could be a ptr returned ++ * from a pointer-cast helper, bpf_sk_fullsock() and ++ * bpf_tcp_sock(). ++ * ++ * Consider the following where "sk" is a reference counted ++ * pointer returned from "sk = bpf_sk_lookup_tcp();": ++ * ++ * 1: sk = bpf_sk_lookup_tcp(); ++ * 2: if (!sk) { return 0; } ++ * 3: fullsock = bpf_sk_fullsock(sk); ++ * 4: if (!fullsock) { bpf_sk_release(sk); return 0; } ++ * 5: tp = bpf_tcp_sock(fullsock); ++ * 6: if (!tp) { bpf_sk_release(sk); return 0; } ++ * 7: bpf_sk_release(sk); ++ * 8: snd_cwnd = tp->snd_cwnd; // verifier will complain ++ * ++ * After bpf_sk_release(sk) at line 7, both "fullsock" ptr and ++ * "tp" ptr should be invalidated also. In order to do that, ++ * the reg holding "fullsock" and "sk" need to remember ++ * the original refcounted ptr id (i.e. sk_reg->id) in ref_obj_id ++ * such that the verifier can reset all regs which have ++ * ref_obj_id matching the sk_reg->id. ++ * ++ * sk_reg->ref_obj_id is set to sk_reg->id at line 1. ++ * sk_reg->id will stay as NULL-marking purpose only. ++ * After NULL-marking is done, sk_reg->id can be reset to 0. ++ * ++ * After "fullsock = bpf_sk_fullsock(sk);" at line 3, ++ * fullsock_reg->ref_obj_id is set to sk_reg->ref_obj_id. ++ * ++ * After "tp = bpf_tcp_sock(fullsock);" at line 5, ++ * tp_reg->ref_obj_id is set to fullsock_reg->ref_obj_id ++ * which is the same as sk_reg->ref_obj_id. ++ * ++ * From the verifier perspective, if sk, fullsock and tp ++ * are not NULL, they are the same ptr with different ++ * reg->type. In particular, bpf_sk_release(tp) is also ++ * allowed and has the same effect as bpf_sk_release(sk). ++ */ ++ u32 ref_obj_id; ++ /* For scalar types (SCALAR_VALUE), this represents our knowledge of ++ * the actual value. ++ * For pointer types, this represents the variable part of the offset ++ * from the pointed-to object, and is shared with all bpf_reg_states ++ * with the same id as us. ++ */ ++ struct tnum var_off; ++ /* Used to determine if any memory access using this register will ++ * result in a bad access. ++ * These refer to the same value as var_off, not necessarily the actual ++ * contents of the register. ++ */ ++ s64 smin_value; /* minimum possible (s64)value */ ++ s64 smax_value; /* maximum possible (s64)value */ ++ u64 umin_value; /* minimum possible (u64)value */ ++ u64 umax_value; /* maximum possible (u64)value */ ++ /* parentage chain for liveness checking */ ++ struct bpf_reg_state *parent; ++ /* Inside the callee two registers can be both PTR_TO_STACK like ++ * R1=fp-8 and R2=fp-8, but one of them points to this function stack ++ * while another to the caller's stack. To differentiate them 'frameno' ++ * is used which is an index in bpf_verifier_state->frame[] array ++ * pointing to bpf_func_state. ++ */ ++ u32 frameno; ++ /* Tracks subreg definition. The stored value is the insn_idx of the ++ * writing insn. This is safe because subreg_def is used before any insn ++ * patching which only happens after main verification finished. ++ */ ++ s32 subreg_def; ++ enum bpf_reg_liveness live; ++ /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ ++ bool precise; ++}; ++ ++enum bpf_stack_slot_type { ++ STACK_INVALID, /* nothing was stored in this stack slot */ ++ STACK_SPILL, /* register spilled into stack */ ++ STACK_MISC, /* BPF program wrote some data into this slot */ ++ STACK_ZERO, /* BPF program wrote constant zero */ ++}; ++ ++#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ ++ ++struct bpf_stack_state { ++ struct bpf_reg_state spilled_ptr; ++ u8 slot_type[BPF_REG_SIZE]; ++}; ++ ++struct bpf_reference_state { ++ /* Track each reference created with a unique id, even if the same ++ * instruction creates the reference multiple times (eg, via CALL). ++ */ ++ int id; ++ /* Instruction where the allocation of this reference occurred. This ++ * is used purely to inform the user of a reference leak. ++ */ ++ int insn_idx; ++}; ++ ++/* state of the program: ++ * type of all registers and stack info ++ */ ++struct bpf_func_state { ++ struct bpf_reg_state regs[MAX_BPF_REG]; ++ /* index of call instruction that called into this func */ ++ int callsite; ++ /* stack frame number of this function state from pov of ++ * enclosing bpf_verifier_state. ++ * 0 = main function, 1 = first callee. ++ */ ++ u32 frameno; ++ /* subprog number == index within subprog_stack_depth ++ * zero == main subprog ++ */ ++ u32 subprogno; ++ ++ /* The following fields should be last. See copy_func_state() */ ++ int acquired_refs; ++ struct bpf_reference_state *refs; ++ int allocated_stack; ++ struct bpf_stack_state *stack; ++}; ++ ++struct bpf_idx_pair { ++ u32 prev_idx; ++ u32 idx; ++}; ++ ++#define MAX_CALL_FRAMES 8 ++struct bpf_verifier_state { ++ /* call stack tracking */ ++ struct bpf_func_state *frame[MAX_CALL_FRAMES]; ++ struct bpf_verifier_state *parent; ++ /* ++ * 'branches' field is the number of branches left to explore: ++ * 0 - all possible paths from this state reached bpf_exit or ++ * were safely pruned ++ * 1 - at least one path is being explored. ++ * This state hasn't reached bpf_exit ++ * 2 - at least two paths are being explored. ++ * This state is an immediate parent of two children. ++ * One is fallthrough branch with branches==1 and another ++ * state is pushed into stack (to be explored later) also with ++ * branches==1. The parent of this state has branches==1. ++ * The verifier state tree connected via 'parent' pointer looks like: ++ * 1 ++ * 1 ++ * 2 -> 1 (first 'if' pushed into stack) ++ * 1 ++ * 2 -> 1 (second 'if' pushed into stack) ++ * 1 ++ * 1 ++ * 1 bpf_exit. ++ * ++ * Once do_check() reaches bpf_exit, it calls update_branch_counts() ++ * and the verifier state tree will look: ++ * 1 ++ * 1 ++ * 2 -> 1 (first 'if' pushed into stack) ++ * 1 ++ * 1 -> 1 (second 'if' pushed into stack) ++ * 0 ++ * 0 ++ * 0 bpf_exit. ++ * After pop_stack() the do_check() will resume at second 'if'. ++ * ++ * If is_state_visited() sees a state with branches > 0 it means ++ * there is a loop. If such state is exactly equal to the current state ++ * it's an infinite loop. Note states_equal() checks for states ++ * equvalency, so two states being 'states_equal' does not mean ++ * infinite loop. The exact comparison is provided by ++ * states_maybe_looping() function. It's a stronger pre-check and ++ * much faster than states_equal(). ++ * ++ * This algorithm may not find all possible infinite loops or ++ * loop iteration count may be too high. ++ * In such cases BPF_COMPLEXITY_LIMIT_INSNS limit kicks in. ++ */ ++ u32 branches; ++ u32 insn_idx; ++ u32 curframe; ++ u32 active_spin_lock; ++ bool speculative; ++ ++ /* first and last insn idx of this verifier state */ ++ u32 first_insn_idx; ++ u32 last_insn_idx; ++ /* jmp history recorded from first to last. ++ * backtracking is using it to go from last to first. ++ * For most states jmp_history_cnt is [0-3]. ++ * For loops can go up to ~40. ++ */ ++ struct bpf_idx_pair *jmp_history; ++ u32 jmp_history_cnt; ++}; ++ ++#define bpf_get_spilled_reg(slot, frame) \ ++ (((slot < frame->allocated_stack / BPF_REG_SIZE) && \ ++ (frame->stack[slot].slot_type[0] == STACK_SPILL)) \ ++ ? &frame->stack[slot].spilled_ptr : NULL) ++ ++/* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ ++#define bpf_for_each_spilled_reg(iter, frame, reg) \ ++ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame); \ ++ iter < frame->allocated_stack / BPF_REG_SIZE; \ ++ iter++, reg = bpf_get_spilled_reg(iter, frame)) ++ ++/* linked list of verifier states used to prune search */ ++struct bpf_verifier_state_list { ++ struct bpf_verifier_state state; ++ struct bpf_verifier_state_list *next; ++ int miss_cnt, hit_cnt; ++}; ++ ++/* Possible states for alu_state member. */ ++#define BPF_ALU_SANITIZE_SRC (1U << 0) ++#define BPF_ALU_SANITIZE_DST (1U << 1) ++#define BPF_ALU_NEG_VALUE (1U << 2) ++#define BPF_ALU_NON_POINTER (1U << 3) ++#define BPF_ALU_IMMEDIATE (1U << 4) ++#define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \ ++ BPF_ALU_SANITIZE_DST) ++ ++struct bpf_insn_aux_data { ++ union { ++ enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ ++ unsigned long map_state; /* pointer/poison value for maps */ ++ s32 call_imm; /* saved imm field of call insn */ ++ u32 alu_limit; /* limit for add/sub register with pointer */ ++ struct { ++ u32 map_index; /* index into used_maps[] */ ++ u32 map_off; /* offset from value base address */ ++ }; ++ }; ++ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ ++ int sanitize_stack_off; /* stack slot to be cleared */ ++ bool seen; /* this insn was processed by the verifier */ ++ bool zext_dst; /* this insn zero extends dst reg */ ++ u8 alu_state; /* used in combination with alu_limit */ ++ bool prune_point; ++ unsigned int orig_idx; /* original instruction index */ ++}; ++ ++#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ ++ ++#define BPF_VERIFIER_TMP_LOG_SIZE 1024 ++ ++struct bpf_verifier_log { ++ u32 level; ++ char kbuf[BPF_VERIFIER_TMP_LOG_SIZE]; ++ char __user *ubuf; ++ u32 len_used; ++ u32 len_total; ++}; ++ ++static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) ++{ ++ return log->len_used >= log->len_total - 1; ++} ++ ++#define BPF_LOG_LEVEL1 1 ++#define BPF_LOG_LEVEL2 2 ++#define BPF_LOG_STATS 4 ++#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2) ++#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS) ++ ++static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) ++{ ++ return log->level && log->ubuf && !bpf_verifier_log_full(log); ++} ++ ++#define BPF_MAX_SUBPROGS 256 ++ ++struct bpf_subprog_info { ++ u32 start; /* insn idx of function entry point */ ++ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ ++ u16 stack_depth; /* max. stack depth used by this function */ ++ bool has_tail_call; ++}; ++ ++/* single container for all structs ++ * one verifier_env per bpf_check() call ++ */ ++struct bpf_verifier_env { ++ u32 insn_idx; ++ u32 prev_insn_idx; ++ struct bpf_prog *prog; /* eBPF program being verified */ ++ const struct bpf_verifier_ops *ops; ++ struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */ ++ int stack_size; /* number of states to be processed */ ++ bool strict_alignment; /* perform strict pointer alignment checks */ ++ bool test_state_freq; /* test verifier with different pruning frequency */ ++ struct bpf_verifier_state *cur_state; /* current verifier state */ ++ struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ ++ struct bpf_verifier_state_list *free_list; ++ struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */ ++ u32 used_map_cnt; /* number of used maps */ ++ u32 id_gen; /* used to generate unique reg IDs */ ++ bool allow_ptr_leaks; ++ bool seen_direct_write; ++ struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ ++ const struct bpf_line_info *prev_linfo; ++ struct bpf_verifier_log log; ++ struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; ++ struct { ++ int *insn_state; ++ int *insn_stack; ++ int cur_stack; ++ } cfg; ++ u32 subprog_cnt; ++ /* number of instructions analyzed by the verifier */ ++ u32 prev_insn_processed, insn_processed; ++ /* number of jmps, calls, exits analyzed so far */ ++ u32 prev_jmps_processed, jmps_processed; ++ /* total verification time */ ++ u64 verification_time; ++ /* maximum number of verifier states kept in 'branching' instructions */ ++ u32 max_states_per_insn; ++ /* total number of allocated verifier states */ ++ u32 total_states; ++ /* some states are freed during program analysis. ++ * this is peak number of states. this number dominates kernel ++ * memory consumption during verification ++ */ ++ u32 peak_states; ++ /* longest register parentage chain walked for liveness marking */ ++ u32 longest_mark_read_walk; ++}; ++ ++__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, ++ const char *fmt, va_list args); ++__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, ++ const char *fmt, ...); ++ ++static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env) ++{ ++ struct bpf_verifier_state *cur = env->cur_state; ++ ++ return cur->frame[cur->curframe]; ++} ++ ++static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env) ++{ ++ return cur_func(env)->regs; ++} ++ ++int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); ++int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, ++ int insn_idx, int prev_insn_idx); ++int bpf_prog_offload_finalize(struct bpf_verifier_env *env); ++void ++bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, ++ struct bpf_insn *insn); ++void ++bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); ++ ++#endif /* _LINUX_BPF_VERIFIER_H */ +--- /dev/null ++++ b/include/linux/btf.h +@@ -0,0 +1,72 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* Copyright (c) 2018 Facebook */ ++ ++#ifndef _LINUX_BTF_H ++#define _LINUX_BTF_H 1 ++ ++#include ++ ++struct btf; ++struct btf_member; ++struct btf_type; ++union bpf_attr; ++ ++extern const struct file_operations btf_fops; ++ ++void btf_put(struct btf *btf); ++int btf_new_fd(const union bpf_attr *attr); ++struct btf *btf_get_by_fd(int fd); ++int btf_get_info_by_fd(const struct btf *btf, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr); ++/* Figure out the size of a type_id. If type_id is a modifier ++ * (e.g. const), it will be resolved to find out the type with size. ++ * ++ * For example: ++ * In describing "const void *", type_id is "const" and "const" ++ * refers to "void *". The return type will be "void *". ++ * ++ * If type_id is a simple "int", then return type will be "int". ++ * ++ * @btf: struct btf object ++ * @type_id: Find out the size of type_id. The type_id of the return ++ * type is set to *type_id. ++ * @ret_size: It can be NULL. If not NULL, the size of the return ++ * type is set to *ret_size. ++ * Return: The btf_type (resolved to another type with size info if needed). ++ * NULL is returned if type_id itself does not have size info ++ * (e.g. void) or it cannot be resolved to another type that ++ * has size info. ++ * *type_id and *ret_size will not be changed in the ++ * NULL return case. ++ */ ++const struct btf_type *btf_type_id_size(const struct btf *btf, ++ u32 *type_id, ++ u32 *ret_size); ++void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, ++ struct seq_file *m); ++int btf_get_fd_by_id(u32 id); ++u32 btf_id(const struct btf *btf); ++bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, ++ const struct btf_member *m, ++ u32 expected_offset, u32 expected_size); ++int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t); ++bool btf_type_is_void(const struct btf_type *t); ++ ++#ifdef CONFIG_BPF_SYSCALL ++const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); ++const char *btf_name_by_offset(const struct btf *btf, u32 offset); ++#else ++static inline const struct btf_type *btf_type_by_id(const struct btf *btf, ++ u32 type_id) ++{ ++ return NULL; ++} ++static inline const char *btf_name_by_offset(const struct btf *btf, ++ u32 offset) ++{ ++ return NULL; ++} ++#endif ++ ++#endif +--- a/include/uapi/linux/bpf_common.h ++++ b/include/uapi/linux/bpf_common.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + #ifndef _UAPI__LINUX_BPF_COMMON_H__ + #define _UAPI__LINUX_BPF_COMMON_H__ + +@@ -14,9 +15,10 @@ + + /* ld/ldx fields */ + #define BPF_SIZE(code) ((code) & 0x18) +-#define BPF_W 0x00 +-#define BPF_H 0x08 +-#define BPF_B 0x10 ++#define BPF_W 0x00 /* 32-bit */ ++#define BPF_H 0x08 /* 16-bit */ ++#define BPF_B 0x10 /* 8-bit */ ++/* eBPF BPF_DW 0x18 64-bit */ + #define BPF_MODE(code) ((code) & 0xe0) + #define BPF_IMM 0x00 + #define BPF_ABS 0x20 +--- a/include/uapi/linux/bpf.h ++++ b/include/uapi/linux/bpf.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com + * + * This program is free software; you can redistribute it and/or +@@ -13,10 +14,11 @@ + /* Extended instruction set based on top of classic BPF */ + + /* instruction classes */ ++#define BPF_JMP32 0x06 /* jmp mode in word width */ + #define BPF_ALU64 0x07 /* alu mode in double word width */ + + /* ld/ldx fields */ +-#define BPF_DW 0x18 /* double word */ ++#define BPF_DW 0x18 /* double word (64-bit) */ + #define BPF_XADD 0xc0 /* exclusive add */ + + /* alu/jmp fields */ +@@ -30,9 +32,14 @@ + #define BPF_FROM_LE BPF_TO_LE + #define BPF_FROM_BE BPF_TO_BE + ++/* jmp encodings */ + #define BPF_JNE 0x50 /* jump != */ ++#define BPF_JLT 0xa0 /* LT is unsigned, '<' */ ++#define BPF_JLE 0xb0 /* LE is unsigned, '<=' */ + #define BPF_JSGT 0x60 /* SGT is signed '>', GT in x86 */ + #define BPF_JSGE 0x70 /* SGE is signed '>=', GE in x86 */ ++#define BPF_JSLT 0xc0 /* SLT is signed, '<' */ ++#define BPF_JSLE 0xd0 /* SLE is signed, '<=' */ + #define BPF_CALL 0x80 /* function call */ + #define BPF_EXIT 0x90 /* function return */ + +@@ -63,6 +70,17 @@ struct bpf_insn { + __s32 imm; /* signed immediate constant */ + }; + ++/* Key of an a BPF_MAP_TYPE_LPM_TRIE entry */ ++struct bpf_lpm_trie_key { ++ __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6 */ ++ __u8 data[0]; /* Arbitrary size */ ++}; ++ ++struct bpf_cgroup_storage_key { ++ __u64 cgroup_inode_id; /* cgroup inode id */ ++ __u32 attach_type; /* program attach type */ ++}; ++ + /* BPF syscall commands, see bpf(2) man-page for details. */ + enum bpf_cmd { + BPF_MAP_CREATE, +@@ -73,6 +91,22 @@ enum bpf_cmd { + BPF_PROG_LOAD, + BPF_OBJ_PIN, + BPF_OBJ_GET, ++ BPF_PROG_ATTACH, ++ BPF_PROG_DETACH, ++ BPF_PROG_TEST_RUN, ++ BPF_PROG_GET_NEXT_ID, ++ BPF_MAP_GET_NEXT_ID, ++ BPF_PROG_GET_FD_BY_ID, ++ BPF_MAP_GET_FD_BY_ID, ++ BPF_OBJ_GET_INFO_BY_FD, ++ BPF_PROG_QUERY, ++ BPF_RAW_TRACEPOINT_OPEN, ++ BPF_BTF_LOAD, ++ BPF_BTF_GET_FD_BY_ID, ++ BPF_TASK_FD_QUERY, ++ BPF_MAP_LOOKUP_AND_DELETE_ELEM, ++ BPF_MAP_FREEZE, ++ BPF_BTF_GET_NEXT_ID, + }; + + enum bpf_map_type { +@@ -81,22 +115,256 @@ enum bpf_map_type { + BPF_MAP_TYPE_ARRAY, + BPF_MAP_TYPE_PROG_ARRAY, + BPF_MAP_TYPE_PERF_EVENT_ARRAY, ++ BPF_MAP_TYPE_PERCPU_HASH, ++ BPF_MAP_TYPE_PERCPU_ARRAY, ++ BPF_MAP_TYPE_STACK_TRACE, ++ BPF_MAP_TYPE_CGROUP_ARRAY, ++ BPF_MAP_TYPE_LRU_HASH, ++ BPF_MAP_TYPE_LRU_PERCPU_HASH, ++ BPF_MAP_TYPE_LPM_TRIE, ++ BPF_MAP_TYPE_ARRAY_OF_MAPS, ++ BPF_MAP_TYPE_HASH_OF_MAPS, ++ BPF_MAP_TYPE_DEVMAP, ++ BPF_MAP_TYPE_SOCKMAP, ++ BPF_MAP_TYPE_CPUMAP, ++ BPF_MAP_TYPE_XSKMAP, ++ BPF_MAP_TYPE_SOCKHASH, ++ BPF_MAP_TYPE_CGROUP_STORAGE, ++ BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, ++ BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, ++ BPF_MAP_TYPE_QUEUE, ++ BPF_MAP_TYPE_STACK, ++ BPF_MAP_TYPE_SK_STORAGE, ++ BPF_MAP_TYPE_DEVMAP_HASH, + }; + ++/* Note that tracing related programs such as ++ * BPF_PROG_TYPE_{KPROBE,TRACEPOINT,PERF_EVENT,RAW_TRACEPOINT} ++ * are not subject to a stable API since kernel internal data ++ * structures can change from release to release and may ++ * therefore break existing tracing BPF programs. Tracing BPF ++ * programs correspond to /a/ specific kernel which is to be ++ * analyzed, and not /a/ specific kernel /and/ all future ones. ++ */ + enum bpf_prog_type { + BPF_PROG_TYPE_UNSPEC, + BPF_PROG_TYPE_SOCKET_FILTER, + BPF_PROG_TYPE_KPROBE, + BPF_PROG_TYPE_SCHED_CLS, + BPF_PROG_TYPE_SCHED_ACT, ++ BPF_PROG_TYPE_TRACEPOINT, ++ BPF_PROG_TYPE_XDP, ++ BPF_PROG_TYPE_PERF_EVENT, ++ BPF_PROG_TYPE_CGROUP_SKB, ++ BPF_PROG_TYPE_CGROUP_SOCK, ++ BPF_PROG_TYPE_LWT_IN, ++ BPF_PROG_TYPE_LWT_OUT, ++ BPF_PROG_TYPE_LWT_XMIT, ++ BPF_PROG_TYPE_SOCK_OPS, ++ BPF_PROG_TYPE_SK_SKB, ++ BPF_PROG_TYPE_CGROUP_DEVICE, ++ BPF_PROG_TYPE_SK_MSG, ++ BPF_PROG_TYPE_RAW_TRACEPOINT, ++ BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ++ BPF_PROG_TYPE_LWT_SEG6LOCAL, ++ BPF_PROG_TYPE_LIRC_MODE2, ++ BPF_PROG_TYPE_SK_REUSEPORT, ++ BPF_PROG_TYPE_FLOW_DISSECTOR, ++ BPF_PROG_TYPE_CGROUP_SYSCTL, ++ BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, ++ BPF_PROG_TYPE_CGROUP_SOCKOPT, + }; + ++enum bpf_attach_type { ++ BPF_CGROUP_INET_INGRESS, ++ BPF_CGROUP_INET_EGRESS, ++ BPF_CGROUP_INET_SOCK_CREATE, ++ BPF_CGROUP_SOCK_OPS, ++ BPF_SK_SKB_STREAM_PARSER, ++ BPF_SK_SKB_STREAM_VERDICT, ++ BPF_CGROUP_DEVICE, ++ BPF_SK_MSG_VERDICT, ++ BPF_CGROUP_INET4_BIND, ++ BPF_CGROUP_INET6_BIND, ++ BPF_CGROUP_INET4_CONNECT, ++ BPF_CGROUP_INET6_CONNECT, ++ BPF_CGROUP_INET4_POST_BIND, ++ BPF_CGROUP_INET6_POST_BIND, ++ BPF_CGROUP_UDP4_SENDMSG, ++ BPF_CGROUP_UDP6_SENDMSG, ++ BPF_LIRC_MODE2, ++ BPF_FLOW_DISSECTOR, ++ BPF_CGROUP_SYSCTL, ++ BPF_CGROUP_UDP4_RECVMSG, ++ BPF_CGROUP_UDP6_RECVMSG, ++ BPF_CGROUP_GETSOCKOPT, ++ BPF_CGROUP_SETSOCKOPT, ++ __MAX_BPF_ATTACH_TYPE ++}; ++ ++#define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE ++ ++/* cgroup-bpf attach flags used in BPF_PROG_ATTACH command ++ * ++ * NONE(default): No further bpf programs allowed in the subtree. ++ * ++ * BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program, ++ * the program in this cgroup yields to sub-cgroup program. ++ * ++ * BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program, ++ * that cgroup program gets run in addition to the program in this cgroup. ++ * ++ * Only one program is allowed to be attached to a cgroup with ++ * NONE or BPF_F_ALLOW_OVERRIDE flag. ++ * Attaching another program on top of NONE or BPF_F_ALLOW_OVERRIDE will ++ * release old program and attach the new one. Attach flags has to match. ++ * ++ * Multiple programs are allowed to be attached to a cgroup with ++ * BPF_F_ALLOW_MULTI flag. They are executed in FIFO order ++ * (those that were attached first, run first) ++ * The programs of sub-cgroup are executed first, then programs of ++ * this cgroup and then programs of parent cgroup. ++ * When children program makes decision (like picking TCP CA or sock bind) ++ * parent program has a chance to override it. ++ * ++ * A cgroup with MULTI or OVERRIDE flag allows any attach flags in sub-cgroups. ++ * A cgroup with NONE doesn't allow any programs in sub-cgroups. ++ * Ex1: ++ * cgrp1 (MULTI progs A, B) -> ++ * cgrp2 (OVERRIDE prog C) -> ++ * cgrp3 (MULTI prog D) -> ++ * cgrp4 (OVERRIDE prog E) -> ++ * cgrp5 (NONE prog F) ++ * the event in cgrp5 triggers execution of F,D,A,B in that order. ++ * if prog F is detached, the execution is E,D,A,B ++ * if prog F and D are detached, the execution is E,A,B ++ * if prog F, E and D are detached, the execution is C,A,B ++ * ++ * All eligible programs are executed regardless of return code from ++ * earlier programs. ++ */ ++#define BPF_F_ALLOW_OVERRIDE (1U << 0) ++#define BPF_F_ALLOW_MULTI (1U << 1) ++ ++/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the ++ * verifier will perform strict alignment checking as if the kernel ++ * has been built with CONFIG_EFFICIENT_UNALIGNED_ACCESS not set, ++ * and NET_IP_ALIGN defined to 2. ++ */ ++#define BPF_F_STRICT_ALIGNMENT (1U << 0) ++ ++/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the ++ * verifier will allow any alignment whatsoever. On platforms ++ * with strict alignment requirements for loads ands stores (such ++ * as sparc and mips) the verifier validates that all loads and ++ * stores provably follow this requirement. This flag turns that ++ * checking and enforcement off. ++ * ++ * It is mostly used for testing when we want to validate the ++ * context and memory access aspects of the verifier, but because ++ * of an unaligned access the alignment check would trigger before ++ * the one we are interested in. ++ */ ++#define BPF_F_ANY_ALIGNMENT (1U << 1) ++ ++/* BPF_F_TEST_RND_HI32 is used in BPF_PROG_LOAD command for testing purpose. ++ * Verifier does sub-register def/use analysis and identifies instructions whose ++ * def only matters for low 32-bit, high 32-bit is never referenced later ++ * through implicit zero extension. Therefore verifier notifies JIT back-ends ++ * that it is safe to ignore clearing high 32-bit for these instructions. This ++ * saves some back-ends a lot of code-gen. However such optimization is not ++ * necessary on some arches, for example x86_64, arm64 etc, whose JIT back-ends ++ * hence hasn't used verifier's analysis result. But, we really want to have a ++ * way to be able to verify the correctness of the described optimization on ++ * x86_64 on which testsuites are frequently exercised. ++ * ++ * So, this flag is introduced. Once it is set, verifier will randomize high ++ * 32-bit for those instructions who has been identified as safe to ignore them. ++ * Then, if verifier is not doing correct analysis, such randomization will ++ * regress tests to expose bugs. ++ */ ++#define BPF_F_TEST_RND_HI32 (1U << 2) ++ ++/* The verifier internal test flag. Behavior is undefined */ ++#define BPF_F_TEST_STATE_FREQ (1U << 3) ++ ++/* When BPF ldimm64's insn[0].src_reg != 0 then this can have ++ * two extensions: ++ * ++ * insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE ++ * insn[0].imm: map fd map fd ++ * insn[1].imm: 0 offset into value ++ * insn[0].off: 0 0 ++ * insn[1].off: 0 0 ++ * ldimm64 rewrite: address of map address of map[0]+offset ++ * verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE ++ */ + #define BPF_PSEUDO_MAP_FD 1 ++#define BPF_PSEUDO_MAP_VALUE 2 ++ ++/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative ++ * offset to another bpf function ++ */ ++#define BPF_PSEUDO_CALL 1 + + /* flags for BPF_MAP_UPDATE_ELEM command */ + #define BPF_ANY 0 /* create new element or update existing */ + #define BPF_NOEXIST 1 /* create new element if it didn't exist */ + #define BPF_EXIST 2 /* update existing element */ ++#define BPF_F_LOCK 4 /* spin_lock-ed map_lookup/map_update */ ++ ++/* flags for BPF_MAP_CREATE command */ ++#define BPF_F_NO_PREALLOC (1U << 0) ++/* Instead of having one common LRU list in the ++ * BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list ++ * which can scale and perform better. ++ * Note, the LRU nodes (including free nodes) cannot be moved ++ * across different LRU lists. ++ */ ++#define BPF_F_NO_COMMON_LRU (1U << 1) ++/* Specify numa node during map creation */ ++#define BPF_F_NUMA_NODE (1U << 2) ++ ++#define BPF_OBJ_NAME_LEN 16U ++ ++/* Flags for accessing BPF object from syscall side. */ ++#define BPF_F_RDONLY (1U << 3) ++#define BPF_F_WRONLY (1U << 4) ++ ++/* Flag for stack_map, store build_id+offset instead of pointer */ ++#define BPF_F_STACK_BUILD_ID (1U << 5) ++ ++/* Zero-initialize hash function seed. This should only be used for testing. */ ++#define BPF_F_ZERO_SEED (1U << 6) ++ ++/* Flags for accessing BPF object from program side. */ ++#define BPF_F_RDONLY_PROG (1U << 7) ++#define BPF_F_WRONLY_PROG (1U << 8) ++ ++/* Clone map from listener for newly accepted socket */ ++#define BPF_F_CLONE (1U << 9) ++ ++/* flags for BPF_PROG_QUERY */ ++#define BPF_F_QUERY_EFFECTIVE (1U << 0) ++ ++enum bpf_stack_build_id_status { ++ /* user space need an empty entry to identify end of a trace */ ++ BPF_STACK_BUILD_ID_EMPTY = 0, ++ /* with valid build_id and offset */ ++ BPF_STACK_BUILD_ID_VALID = 1, ++ /* couldn't get build_id, fallback to ip */ ++ BPF_STACK_BUILD_ID_IP = 2, ++}; ++ ++#define BPF_BUILD_ID_SIZE 20 ++struct bpf_stack_build_id { ++ __s32 status; ++ unsigned char build_id[BPF_BUILD_ID_SIZE]; ++ union { ++ __u64 offset; ++ __u64 ip; ++ }; ++}; + + union bpf_attr { + struct { /* anonymous struct used by BPF_MAP_CREATE command */ +@@ -104,6 +372,18 @@ union bpf_attr { + __u32 key_size; /* size of key in bytes */ + __u32 value_size; /* size of value in bytes */ + __u32 max_entries; /* max number of entries in a map */ ++ __u32 map_flags; /* BPF_MAP_CREATE related ++ * flags defined above. ++ */ ++ __u32 inner_map_fd; /* fd pointing to the inner map */ ++ __u32 numa_node; /* numa node (effective only if ++ * BPF_F_NUMA_NODE is set). ++ */ ++ char map_name[BPF_OBJ_NAME_LEN]; ++ __u32 map_ifindex; /* ifindex of netdev to create on */ ++ __u32 btf_fd; /* fd pointing to a BTF type data */ ++ __u32 btf_key_type_id; /* BTF type_id of the key */ ++ __u32 btf_value_type_id; /* BTF type_id of the value */ + }; + + struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ +@@ -124,154 +404,2568 @@ union bpf_attr { + __u32 log_level; /* verbosity level of verifier */ + __u32 log_size; /* size of user buffer */ + __aligned_u64 log_buf; /* user supplied buffer */ +- __u32 kern_version; /* checked when prog_type=kprobe */ ++ __u32 kern_version; /* not used */ ++ __u32 prog_flags; ++ char prog_name[BPF_OBJ_NAME_LEN]; ++ __u32 prog_ifindex; /* ifindex of netdev to prep for */ ++ /* For some prog types expected attach type must be known at ++ * load time to verify attach type specific parts of prog ++ * (context accesses, allowed helpers, etc). ++ */ ++ __u32 expected_attach_type; ++ __u32 prog_btf_fd; /* fd pointing to BTF type data */ ++ __u32 func_info_rec_size; /* userspace bpf_func_info size */ ++ __aligned_u64 func_info; /* func info */ ++ __u32 func_info_cnt; /* number of bpf_func_info records */ ++ __u32 line_info_rec_size; /* userspace bpf_line_info size */ ++ __aligned_u64 line_info; /* line info */ ++ __u32 line_info_cnt; /* number of bpf_line_info records */ + }; + + struct { /* anonymous struct used by BPF_OBJ_* commands */ + __aligned_u64 pathname; + __u32 bpf_fd; ++ __u32 file_flags; ++ }; ++ ++ struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ ++ __u32 target_fd; /* container object to attach to */ ++ __u32 attach_bpf_fd; /* eBPF program to attach */ ++ __u32 attach_type; ++ __u32 attach_flags; ++ }; ++ ++ struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ ++ __u32 prog_fd; ++ __u32 retval; ++ __u32 data_size_in; /* input: len of data_in */ ++ __u32 data_size_out; /* input/output: len of data_out ++ * returns ENOSPC if data_out ++ * is too small. ++ */ ++ __aligned_u64 data_in; ++ __aligned_u64 data_out; ++ __u32 repeat; ++ __u32 duration; ++ __u32 ctx_size_in; /* input: len of ctx_in */ ++ __u32 ctx_size_out; /* input/output: len of ctx_out ++ * returns ENOSPC if ctx_out ++ * is too small. ++ */ ++ __aligned_u64 ctx_in; ++ __aligned_u64 ctx_out; ++ } test; ++ ++ struct { /* anonymous struct used by BPF_*_GET_*_ID */ ++ union { ++ __u32 start_id; ++ __u32 prog_id; ++ __u32 map_id; ++ __u32 btf_id; ++ }; ++ __u32 next_id; ++ __u32 open_flags; + }; ++ ++ struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ ++ __u32 bpf_fd; ++ __u32 info_len; ++ __aligned_u64 info; ++ } info; ++ ++ struct { /* anonymous struct used by BPF_PROG_QUERY command */ ++ __u32 target_fd; /* container object to query */ ++ __u32 attach_type; ++ __u32 query_flags; ++ __u32 attach_flags; ++ __aligned_u64 prog_ids; ++ __u32 prog_cnt; ++ } query; ++ ++ struct { ++ __u64 name; ++ __u32 prog_fd; ++ } raw_tracepoint; ++ ++ struct { /* anonymous struct for BPF_BTF_LOAD */ ++ __aligned_u64 btf; ++ __aligned_u64 btf_log_buf; ++ __u32 btf_size; ++ __u32 btf_log_size; ++ __u32 btf_log_level; ++ }; ++ ++ struct { ++ __u32 pid; /* input: pid */ ++ __u32 fd; /* input: fd */ ++ __u32 flags; /* input: flags */ ++ __u32 buf_len; /* input/output: buf len */ ++ __aligned_u64 buf; /* input/output: ++ * tp_name for tracepoint ++ * symbol for kprobe ++ * filename for uprobe ++ */ ++ __u32 prog_id; /* output: prod_id */ ++ __u32 fd_type; /* output: BPF_FD_TYPE_* */ ++ __u64 probe_offset; /* output: probe_offset */ ++ __u64 probe_addr; /* output: probe_addr */ ++ } task_fd_query; + } __attribute__((aligned(8))); + ++/* The description below is an attempt at providing documentation to eBPF ++ * developers about the multiple available eBPF helper functions. It can be ++ * parsed and used to produce a manual page. The workflow is the following, ++ * and requires the rst2man utility: ++ * ++ * $ ./scripts/bpf_helpers_doc.py \ ++ * --filename include/uapi/linux/bpf.h > /tmp/bpf-helpers.rst ++ * $ rst2man /tmp/bpf-helpers.rst > /tmp/bpf-helpers.7 ++ * $ man /tmp/bpf-helpers.7 ++ * ++ * Note that in order to produce this external documentation, some RST ++ * formatting is used in the descriptions to get "bold" and "italics" in ++ * manual pages. Also note that the few trailing white spaces are ++ * intentional, removing them would break paragraphs for rst2man. ++ * ++ * Start of BPF helper function descriptions: ++ * ++ * void *bpf_map_lookup_elem(struct bpf_map *map, const void *key) ++ * Description ++ * Perform a lookup in *map* for an entry associated to *key*. ++ * Return ++ * Map value associated to *key*, or **NULL** if no entry was ++ * found. ++ * ++ * int bpf_map_update_elem(struct bpf_map *map, const void *key, const void *value, u64 flags) ++ * Description ++ * Add or update the value of the entry associated to *key* in ++ * *map* with *value*. *flags* is one of: ++ * ++ * **BPF_NOEXIST** ++ * The entry for *key* must not exist in the map. ++ * **BPF_EXIST** ++ * The entry for *key* must already exist in the map. ++ * **BPF_ANY** ++ * No condition on the existence of the entry for *key*. ++ * ++ * Flag value **BPF_NOEXIST** cannot be used for maps of types ++ * **BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY** (all ++ * elements always exist), the helper would return an error. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_map_delete_elem(struct bpf_map *map, const void *key) ++ * Description ++ * Delete entry with *key* from *map*. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_probe_read(void *dst, u32 size, const void *src) ++ * Description ++ * For tracing programs, safely attempt to read *size* bytes from ++ * address *src* and store the data in *dst*. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * u64 bpf_ktime_get_ns(void) ++ * Description ++ * Return the time elapsed since system boot, in nanoseconds. ++ * Return ++ * Current *ktime*. ++ * ++ * int bpf_trace_printk(const char *fmt, u32 fmt_size, ...) ++ * Description ++ * This helper is a "printk()-like" facility for debugging. It ++ * prints a message defined by format *fmt* (of size *fmt_size*) ++ * to file *\/sys/kernel/debug/tracing/trace* from DebugFS, if ++ * available. It can take up to three additional **u64** ++ * arguments (as an eBPF helpers, the total number of arguments is ++ * limited to five). ++ * ++ * Each time the helper is called, it appends a line to the trace. ++ * Lines are discarded while *\/sys/kernel/debug/tracing/trace* is ++ * open, use *\/sys/kernel/debug/tracing/trace_pipe* to avoid this. ++ * The format of the trace is customizable, and the exact output ++ * one will get depends on the options set in ++ * *\/sys/kernel/debug/tracing/trace_options* (see also the ++ * *README* file under the same directory). However, it usually ++ * defaults to something like: ++ * ++ * :: ++ * ++ * telnet-470 [001] .N.. 419421.045894: 0x00000001: ++ * ++ * In the above: ++ * ++ * * ``telnet`` is the name of the current task. ++ * * ``470`` is the PID of the current task. ++ * * ``001`` is the CPU number on which the task is ++ * running. ++ * * In ``.N..``, each character refers to a set of ++ * options (whether irqs are enabled, scheduling ++ * options, whether hard/softirqs are running, level of ++ * preempt_disabled respectively). **N** means that ++ * **TIF_NEED_RESCHED** and **PREEMPT_NEED_RESCHED** ++ * are set. ++ * * ``419421.045894`` is a timestamp. ++ * * ``0x00000001`` is a fake value used by BPF for the ++ * instruction pointer register. ++ * * ```` is the message formatted with ++ * *fmt*. ++ * ++ * The conversion specifiers supported by *fmt* are similar, but ++ * more limited than for printk(). They are **%d**, **%i**, ++ * **%u**, **%x**, **%ld**, **%li**, **%lu**, **%lx**, **%lld**, ++ * **%lli**, **%llu**, **%llx**, **%p**, **%s**. No modifier (size ++ * of field, padding with zeroes, etc.) is available, and the ++ * helper will return **-EINVAL** (but print nothing) if it ++ * encounters an unknown specifier. ++ * ++ * Also, note that **bpf_trace_printk**\ () is slow, and should ++ * only be used for debugging purposes. For this reason, a notice ++ * bloc (spanning several lines) is printed to kernel logs and ++ * states that the helper should not be used "for production use" ++ * the first time this helper is used (or more precisely, when ++ * **trace_printk**\ () buffers are allocated). For passing values ++ * to user space, perf events should be preferred. ++ * Return ++ * The number of bytes written to the buffer, or a negative error ++ * in case of failure. ++ * ++ * u32 bpf_get_prandom_u32(void) ++ * Description ++ * Get a pseudo-random number. ++ * ++ * From a security point of view, this helper uses its own ++ * pseudo-random internal state, and cannot be used to infer the ++ * seed of other random functions in the kernel. However, it is ++ * essential to note that the generator used by the helper is not ++ * cryptographically secure. ++ * Return ++ * A random 32-bit unsigned value. ++ * ++ * u32 bpf_get_smp_processor_id(void) ++ * Description ++ * Get the SMP (symmetric multiprocessing) processor id. Note that ++ * all programs run with preemption disabled, which means that the ++ * SMP processor id is stable during all the execution of the ++ * program. ++ * Return ++ * The SMP id of the processor running the program. ++ * ++ * int bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags) ++ * Description ++ * Store *len* bytes from address *from* into the packet ++ * associated to *skb*, at *offset*. *flags* are a combination of ++ * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the ++ * checksum for the packet after storing the bytes) and ++ * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\ ++ * **->swhash** and *skb*\ **->l4hash** to 0). ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_l3_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 size) ++ * Description ++ * Recompute the layer 3 (e.g. IP) checksum for the packet ++ * associated to *skb*. Computation is incremental, so the helper ++ * must know the former value of the header field that was ++ * modified (*from*), the new value of this field (*to*), and the ++ * number of bytes (2 or 4) for this field, stored in *size*. ++ * Alternatively, it is possible to store the difference between ++ * the previous and the new values of the header field in *to*, by ++ * setting *from* and *size* to 0. For both methods, *offset* ++ * indicates the location of the IP checksum within the packet. ++ * ++ * This helper works in combination with **bpf_csum_diff**\ (), ++ * which does not update the checksum in-place, but offers more ++ * flexibility and can handle sizes larger than 2 or 4 for the ++ * checksum to update. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_l4_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 flags) ++ * Description ++ * Recompute the layer 4 (e.g. TCP, UDP or ICMP) checksum for the ++ * packet associated to *skb*. Computation is incremental, so the ++ * helper must know the former value of the header field that was ++ * modified (*from*), the new value of this field (*to*), and the ++ * number of bytes (2 or 4) for this field, stored on the lowest ++ * four bits of *flags*. Alternatively, it is possible to store ++ * the difference between the previous and the new values of the ++ * header field in *to*, by setting *from* and the four lowest ++ * bits of *flags* to 0. For both methods, *offset* indicates the ++ * location of the IP checksum within the packet. In addition to ++ * the size of the field, *flags* can be added (bitwise OR) actual ++ * flags. With **BPF_F_MARK_MANGLED_0**, a null checksum is left ++ * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and ++ * for updates resulting in a null checksum the value is set to ++ * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates ++ * the checksum is to be computed against a pseudo-header. ++ * ++ * This helper works in combination with **bpf_csum_diff**\ (), ++ * which does not update the checksum in-place, but offers more ++ * flexibility and can handle sizes larger than 2 or 4 for the ++ * checksum to update. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_tail_call(void *ctx, struct bpf_map *prog_array_map, u32 index) ++ * Description ++ * This special helper is used to trigger a "tail call", or in ++ * other words, to jump into another eBPF program. The same stack ++ * frame is used (but values on stack and in registers for the ++ * caller are not accessible to the callee). This mechanism allows ++ * for program chaining, either for raising the maximum number of ++ * available eBPF instructions, or to execute given programs in ++ * conditional blocks. For security reasons, there is an upper ++ * limit to the number of successive tail calls that can be ++ * performed. ++ * ++ * Upon call of this helper, the program attempts to jump into a ++ * program referenced at index *index* in *prog_array_map*, a ++ * special map of type **BPF_MAP_TYPE_PROG_ARRAY**, and passes ++ * *ctx*, a pointer to the context. ++ * ++ * If the call succeeds, the kernel immediately runs the first ++ * instruction of the new program. This is not a function call, ++ * and it never returns to the previous program. If the call ++ * fails, then the helper has no effect, and the caller continues ++ * to run its subsequent instructions. A call can fail if the ++ * destination program for the jump does not exist (i.e. *index* ++ * is superior to the number of entries in *prog_array_map*), or ++ * if the maximum number of tail calls has been reached for this ++ * chain of programs. This limit is defined in the kernel by the ++ * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), ++ * which is currently set to 32. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_clone_redirect(struct sk_buff *skb, u32 ifindex, u64 flags) ++ * Description ++ * Clone and redirect the packet associated to *skb* to another ++ * net device of index *ifindex*. Both ingress and egress ++ * interfaces can be used for redirection. The **BPF_F_INGRESS** ++ * value in *flags* is used to make the distinction (ingress path ++ * is selected if the flag is present, egress path otherwise). ++ * This is the only flag supported for now. ++ * ++ * In comparison with **bpf_redirect**\ () helper, ++ * **bpf_clone_redirect**\ () has the associated cost of ++ * duplicating the packet buffer, but this can be executed out of ++ * the eBPF program. Conversely, **bpf_redirect**\ () is more ++ * efficient, but it is handled through an action code where the ++ * redirection happens only after the eBPF program has returned. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * u64 bpf_get_current_pid_tgid(void) ++ * Return ++ * A 64-bit integer containing the current tgid and pid, and ++ * created as such: ++ * *current_task*\ **->tgid << 32 \|** ++ * *current_task*\ **->pid**. ++ * ++ * u64 bpf_get_current_uid_gid(void) ++ * Return ++ * A 64-bit integer containing the current GID and UID, and ++ * created as such: *current_gid* **<< 32 \|** *current_uid*. ++ * ++ * int bpf_get_current_comm(char *buf, u32 size_of_buf) ++ * Description ++ * Copy the **comm** attribute of the current task into *buf* of ++ * *size_of_buf*. The **comm** attribute contains the name of ++ * the executable (excluding the path) for the current task. The ++ * *size_of_buf* must be strictly positive. On success, the ++ * helper makes sure that the *buf* is NUL-terminated. On failure, ++ * it is filled with zeroes. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * u32 bpf_get_cgroup_classid(struct sk_buff *skb) ++ * Description ++ * Retrieve the classid for the current task, i.e. for the net_cls ++ * cgroup to which *skb* belongs. ++ * ++ * This helper can be used on TC egress path, but not on ingress. ++ * ++ * The net_cls cgroup provides an interface to tag network packets ++ * based on a user-provided identifier for all traffic coming from ++ * the tasks belonging to the related cgroup. See also the related ++ * kernel documentation, available from the Linux sources in file ++ * *Documentation/admin-guide/cgroup-v1/net_cls.rst*. ++ * ++ * The Linux kernel has two versions for cgroups: there are ++ * cgroups v1 and cgroups v2. Both are available to users, who can ++ * use a mixture of them, but note that the net_cls cgroup is for ++ * cgroup v1 only. This makes it incompatible with BPF programs ++ * run on cgroups, which is a cgroup-v2-only feature (a socket can ++ * only hold data for one version of cgroups at a time). ++ * ++ * This helper is only available is the kernel was compiled with ++ * the **CONFIG_CGROUP_NET_CLASSID** configuration option set to ++ * "**y**" or to "**m**". ++ * Return ++ * The classid, or 0 for the default unconfigured classid. ++ * ++ * int bpf_skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) ++ * Description ++ * Push a *vlan_tci* (VLAN tag control information) of protocol ++ * *vlan_proto* to the packet associated to *skb*, then update ++ * the checksum. Note that if *vlan_proto* is different from ++ * **ETH_P_8021Q** and **ETH_P_8021AD**, it is considered to ++ * be **ETH_P_8021Q**. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_vlan_pop(struct sk_buff *skb) ++ * Description ++ * Pop a VLAN header from the packet associated to *skb*. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_get_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) ++ * Description ++ * Get tunnel metadata. This helper takes a pointer *key* to an ++ * empty **struct bpf_tunnel_key** of **size**, that will be ++ * filled with tunnel metadata for the packet associated to *skb*. ++ * The *flags* can be set to **BPF_F_TUNINFO_IPV6**, which ++ * indicates that the tunnel is based on IPv6 protocol instead of ++ * IPv4. ++ * ++ * The **struct bpf_tunnel_key** is an object that generalizes the ++ * principal parameters used by various tunneling protocols into a ++ * single struct. This way, it can be used to easily make a ++ * decision based on the contents of the encapsulation header, ++ * "summarized" in this struct. In particular, it holds the IP ++ * address of the remote end (IPv4 or IPv6, depending on the case) ++ * in *key*\ **->remote_ipv4** or *key*\ **->remote_ipv6**. Also, ++ * this struct exposes the *key*\ **->tunnel_id**, which is ++ * generally mapped to a VNI (Virtual Network Identifier), making ++ * it programmable together with the **bpf_skb_set_tunnel_key**\ ++ * () helper. ++ * ++ * Let's imagine that the following code is part of a program ++ * attached to the TC ingress interface, on one end of a GRE ++ * tunnel, and is supposed to filter out all messages coming from ++ * remote ends with IPv4 address other than 10.0.0.1: ++ * ++ * :: ++ * ++ * int ret; ++ * struct bpf_tunnel_key key = {}; ++ * ++ * ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); ++ * if (ret < 0) ++ * return TC_ACT_SHOT; // drop packet ++ * ++ * if (key.remote_ipv4 != 0x0a000001) ++ * return TC_ACT_SHOT; // drop packet ++ * ++ * return TC_ACT_OK; // accept packet ++ * ++ * This interface can also be used with all encapsulation devices ++ * that can operate in "collect metadata" mode: instead of having ++ * one network device per specific configuration, the "collect ++ * metadata" mode only requires a single device where the ++ * configuration can be extracted from this helper. ++ * ++ * This can be used together with various tunnels such as VXLan, ++ * Geneve, GRE or IP in IP (IPIP). ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_set_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) ++ * Description ++ * Populate tunnel metadata for packet associated to *skb.* The ++ * tunnel metadata is set to the contents of *key*, of *size*. The ++ * *flags* can be set to a combination of the following values: ++ * ++ * **BPF_F_TUNINFO_IPV6** ++ * Indicate that the tunnel is based on IPv6 protocol ++ * instead of IPv4. ++ * **BPF_F_ZERO_CSUM_TX** ++ * For IPv4 packets, add a flag to tunnel metadata ++ * indicating that checksum computation should be skipped ++ * and checksum set to zeroes. ++ * **BPF_F_DONT_FRAGMENT** ++ * Add a flag to tunnel metadata indicating that the ++ * packet should not be fragmented. ++ * **BPF_F_SEQ_NUMBER** ++ * Add a flag to tunnel metadata indicating that a ++ * sequence number should be added to tunnel header before ++ * sending the packet. This flag was added for GRE ++ * encapsulation, but might be used with other protocols ++ * as well in the future. ++ * ++ * Here is a typical usage on the transmit path: ++ * ++ * :: ++ * ++ * struct bpf_tunnel_key key; ++ * populate key ... ++ * bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0); ++ * bpf_clone_redirect(skb, vxlan_dev_ifindex, 0); ++ * ++ * See also the description of the **bpf_skb_get_tunnel_key**\ () ++ * helper for additional information. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * u64 bpf_perf_event_read(struct bpf_map *map, u64 flags) ++ * Description ++ * Read the value of a perf event counter. This helper relies on a ++ * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of ++ * the perf event counter is selected when *map* is updated with ++ * perf event file descriptors. The *map* is an array whose size ++ * is the number of available CPUs, and each cell contains a value ++ * relative to one CPU. The value to retrieve is indicated by ++ * *flags*, that contains the index of the CPU to look up, masked ++ * with **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to ++ * **BPF_F_CURRENT_CPU** to indicate that the value for the ++ * current CPU should be retrieved. ++ * ++ * Note that before Linux 4.13, only hardware perf event can be ++ * retrieved. ++ * ++ * Also, be aware that the newer helper ++ * **bpf_perf_event_read_value**\ () is recommended over ++ * **bpf_perf_event_read**\ () in general. The latter has some ABI ++ * quirks where error and counter value are used as a return code ++ * (which is wrong to do since ranges may overlap). This issue is ++ * fixed with **bpf_perf_event_read_value**\ (), which at the same ++ * time provides more features over the **bpf_perf_event_read**\ ++ * () interface. Please refer to the description of ++ * **bpf_perf_event_read_value**\ () for details. ++ * Return ++ * The value of the perf event counter read from the map, or a ++ * negative error code in case of failure. ++ * ++ * int bpf_redirect(u32 ifindex, u64 flags) ++ * Description ++ * Redirect the packet to another net device of index *ifindex*. ++ * This helper is somewhat similar to **bpf_clone_redirect**\ ++ * (), except that the packet is not cloned, which provides ++ * increased performance. ++ * ++ * Except for XDP, both ingress and egress interfaces can be used ++ * for redirection. The **BPF_F_INGRESS** value in *flags* is used ++ * to make the distinction (ingress path is selected if the flag ++ * is present, egress path otherwise). Currently, XDP only ++ * supports redirection to the egress interface, and accepts no ++ * flag at all. ++ * ++ * The same effect can be attained with the more generic ++ * **bpf_redirect_map**\ (), which requires specific maps to be ++ * used but offers better performance. ++ * Return ++ * For XDP, the helper returns **XDP_REDIRECT** on success or ++ * **XDP_ABORTED** on error. For other program types, the values ++ * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on ++ * error. ++ * ++ * u32 bpf_get_route_realm(struct sk_buff *skb) ++ * Description ++ * Retrieve the realm or the route, that is to say the ++ * **tclassid** field of the destination for the *skb*. The ++ * indentifier retrieved is a user-provided tag, similar to the ++ * one used with the net_cls cgroup (see description for ++ * **bpf_get_cgroup_classid**\ () helper), but here this tag is ++ * held by a route (a destination entry), not by a task. ++ * ++ * Retrieving this identifier works with the clsact TC egress hook ++ * (see also **tc-bpf(8)**), or alternatively on conventional ++ * classful egress qdiscs, but not on TC ingress path. In case of ++ * clsact TC egress hook, this has the advantage that, internally, ++ * the destination entry has not been dropped yet in the transmit ++ * path. Therefore, the destination entry does not need to be ++ * artificially held via **netif_keep_dst**\ () for a classful ++ * qdisc until the *skb* is freed. ++ * ++ * This helper is available only if the kernel was compiled with ++ * **CONFIG_IP_ROUTE_CLASSID** configuration option. ++ * Return ++ * The realm of the route for the packet associated to *skb*, or 0 ++ * if none was found. ++ * ++ * int bpf_perf_event_output(struct pt_regs *ctx, struct bpf_map *map, u64 flags, void *data, u64 size) ++ * Description ++ * Write raw *data* blob into a special BPF perf event held by ++ * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf ++ * event must have the following attributes: **PERF_SAMPLE_RAW** ++ * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and ++ * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. ++ * ++ * The *flags* are used to indicate the index in *map* for which ++ * the value must be put, masked with **BPF_F_INDEX_MASK**. ++ * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** ++ * to indicate that the index of the current CPU core should be ++ * used. ++ * ++ * The value to write, of *size*, is passed through eBPF stack and ++ * pointed by *data*. ++ * ++ * The context of the program *ctx* needs also be passed to the ++ * helper. ++ * ++ * On user space, a program willing to read the values needs to ++ * call **perf_event_open**\ () on the perf event (either for ++ * one or for all CPUs) and to store the file descriptor into the ++ * *map*. This must be done before the eBPF program can send data ++ * into it. An example is available in file ++ * *samples/bpf/trace_output_user.c* in the Linux kernel source ++ * tree (the eBPF program counterpart is in ++ * *samples/bpf/trace_output_kern.c*). ++ * ++ * **bpf_perf_event_output**\ () achieves better performance ++ * than **bpf_trace_printk**\ () for sharing data with user ++ * space, and is much better suitable for streaming data from eBPF ++ * programs. ++ * ++ * Note that this helper is not restricted to tracing use cases ++ * and can be used with programs attached to TC or XDP as well, ++ * where it allows for passing data to user space listeners. Data ++ * can be: ++ * ++ * * Only custom structs, ++ * * Only the packet payload, or ++ * * A combination of both. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len) ++ * Description ++ * This helper was provided as an easy way to load data from a ++ * packet. It can be used to load *len* bytes from *offset* from ++ * the packet associated to *skb*, into the buffer pointed by ++ * *to*. ++ * ++ * Since Linux 4.7, usage of this helper has mostly been replaced ++ * by "direct packet access", enabling packet data to be ++ * manipulated with *skb*\ **->data** and *skb*\ **->data_end** ++ * pointing respectively to the first byte of packet data and to ++ * the byte after the last byte of packet data. However, it ++ * remains useful if one wishes to read large quantities of data ++ * at once from a packet into the eBPF stack. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_get_stackid(struct pt_regs *ctx, struct bpf_map *map, u64 flags) ++ * Description ++ * Walk a user or a kernel stack and return its id. To achieve ++ * this, the helper needs *ctx*, which is a pointer to the context ++ * on which the tracing program is executed, and a pointer to a ++ * *map* of type **BPF_MAP_TYPE_STACK_TRACE**. ++ * ++ * The last argument, *flags*, holds the number of stack frames to ++ * skip (from 0 to 255), masked with ++ * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set ++ * a combination of the following flags: ++ * ++ * **BPF_F_USER_STACK** ++ * Collect a user space stack instead of a kernel stack. ++ * **BPF_F_FAST_STACK_CMP** ++ * Compare stacks by hash only. ++ * **BPF_F_REUSE_STACKID** ++ * If two different stacks hash into the same *stackid*, ++ * discard the old one. ++ * ++ * The stack id retrieved is a 32 bit long integer handle which ++ * can be further combined with other data (including other stack ++ * ids) and used as a key into maps. This can be useful for ++ * generating a variety of graphs (such as flame graphs or off-cpu ++ * graphs). ++ * ++ * For walking a stack, this helper is an improvement over ++ * **bpf_probe_read**\ (), which can be used with unrolled loops ++ * but is not efficient and consumes a lot of eBPF instructions. ++ * Instead, **bpf_get_stackid**\ () can collect up to ++ * **PERF_MAX_STACK_DEPTH** both kernel and user frames. Note that ++ * this limit can be controlled with the **sysctl** program, and ++ * that it should be manually increased in order to profile long ++ * user stacks (such as stacks for Java programs). To do so, use: ++ * ++ * :: ++ * ++ * # sysctl kernel.perf_event_max_stack= ++ * Return ++ * The positive or null stack id on success, or a negative error ++ * in case of failure. ++ * ++ * s64 bpf_csum_diff(__be32 *from, u32 from_size, __be32 *to, u32 to_size, __wsum seed) ++ * Description ++ * Compute a checksum difference, from the raw buffer pointed by ++ * *from*, of length *from_size* (that must be a multiple of 4), ++ * towards the raw buffer pointed by *to*, of size *to_size* ++ * (same remark). An optional *seed* can be added to the value ++ * (this can be cascaded, the seed may come from a previous call ++ * to the helper). ++ * ++ * This is flexible enough to be used in several ways: ++ * ++ * * With *from_size* == 0, *to_size* > 0 and *seed* set to ++ * checksum, it can be used when pushing new data. ++ * * With *from_size* > 0, *to_size* == 0 and *seed* set to ++ * checksum, it can be used when removing data from a packet. ++ * * With *from_size* > 0, *to_size* > 0 and *seed* set to 0, it ++ * can be used to compute a diff. Note that *from_size* and ++ * *to_size* do not need to be equal. ++ * ++ * This helper can be used in combination with ++ * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ (), to ++ * which one can feed in the difference computed with ++ * **bpf_csum_diff**\ (). ++ * Return ++ * The checksum result, or a negative error code in case of ++ * failure. ++ * ++ * int bpf_skb_get_tunnel_opt(struct sk_buff *skb, u8 *opt, u32 size) ++ * Description ++ * Retrieve tunnel options metadata for the packet associated to ++ * *skb*, and store the raw tunnel option data to the buffer *opt* ++ * of *size*. ++ * ++ * This helper can be used with encapsulation devices that can ++ * operate in "collect metadata" mode (please refer to the related ++ * note in the description of **bpf_skb_get_tunnel_key**\ () for ++ * more details). A particular example where this can be used is ++ * in combination with the Geneve encapsulation protocol, where it ++ * allows for pushing (with **bpf_skb_get_tunnel_opt**\ () helper) ++ * and retrieving arbitrary TLVs (Type-Length-Value headers) from ++ * the eBPF program. This allows for full customization of these ++ * headers. ++ * Return ++ * The size of the option data retrieved. ++ * ++ * int bpf_skb_set_tunnel_opt(struct sk_buff *skb, u8 *opt, u32 size) ++ * Description ++ * Set tunnel options metadata for the packet associated to *skb* ++ * to the option data contained in the raw buffer *opt* of *size*. ++ * ++ * See also the description of the **bpf_skb_get_tunnel_opt**\ () ++ * helper for additional information. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_change_proto(struct sk_buff *skb, __be16 proto, u64 flags) ++ * Description ++ * Change the protocol of the *skb* to *proto*. Currently ++ * supported are transition from IPv4 to IPv6, and from IPv6 to ++ * IPv4. The helper takes care of the groundwork for the ++ * transition, including resizing the socket buffer. The eBPF ++ * program is expected to fill the new headers, if any, via ++ * **skb_store_bytes**\ () and to recompute the checksums with ++ * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ ++ * (). The main case for this helper is to perform NAT64 ++ * operations out of an eBPF program. ++ * ++ * Internally, the GSO type is marked as dodgy so that headers are ++ * checked and segments are recalculated by the GSO/GRO engine. ++ * The size for GSO target is adapted as well. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_change_type(struct sk_buff *skb, u32 type) ++ * Description ++ * Change the packet type for the packet associated to *skb*. This ++ * comes down to setting *skb*\ **->pkt_type** to *type*, except ++ * the eBPF program does not have a write access to *skb*\ ++ * **->pkt_type** beside this helper. Using a helper here allows ++ * for graceful handling of errors. ++ * ++ * The major use case is to change incoming *skb*s to ++ * **PACKET_HOST** in a programmatic way instead of having to ++ * recirculate via **redirect**\ (..., **BPF_F_INGRESS**), for ++ * example. ++ * ++ * Note that *type* only allows certain values. At this time, they ++ * are: ++ * ++ * **PACKET_HOST** ++ * Packet is for us. ++ * **PACKET_BROADCAST** ++ * Send packet to all. ++ * **PACKET_MULTICAST** ++ * Send packet to group. ++ * **PACKET_OTHERHOST** ++ * Send packet to someone else. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_under_cgroup(struct sk_buff *skb, struct bpf_map *map, u32 index) ++ * Description ++ * Check whether *skb* is a descendant of the cgroup2 held by ++ * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. ++ * Return ++ * The return value depends on the result of the test, and can be: ++ * ++ * * 0, if the *skb* failed the cgroup2 descendant test. ++ * * 1, if the *skb* succeeded the cgroup2 descendant test. ++ * * A negative error code, if an error occurred. ++ * ++ * u32 bpf_get_hash_recalc(struct sk_buff *skb) ++ * Description ++ * Retrieve the hash of the packet, *skb*\ **->hash**. If it is ++ * not set, in particular if the hash was cleared due to mangling, ++ * recompute this hash. Later accesses to the hash can be done ++ * directly with *skb*\ **->hash**. ++ * ++ * Calling **bpf_set_hash_invalid**\ (), changing a packet ++ * prototype with **bpf_skb_change_proto**\ (), or calling ++ * **bpf_skb_store_bytes**\ () with the ++ * **BPF_F_INVALIDATE_HASH** are actions susceptible to clear ++ * the hash and to trigger a new computation for the next call to ++ * **bpf_get_hash_recalc**\ (). ++ * Return ++ * The 32-bit hash. ++ * ++ * u64 bpf_get_current_task(void) ++ * Return ++ * A pointer to the current task struct. ++ * ++ * int bpf_probe_write_user(void *dst, const void *src, u32 len) ++ * Description ++ * Attempt in a safe way to write *len* bytes from the buffer ++ * *src* to *dst* in memory. It only works for threads that are in ++ * user context, and *dst* must be a valid user space address. ++ * ++ * This helper should not be used to implement any kind of ++ * security mechanism because of TOC-TOU attacks, but rather to ++ * debug, divert, and manipulate execution of semi-cooperative ++ * processes. ++ * ++ * Keep in mind that this feature is meant for experiments, and it ++ * has a risk of crashing the system and running programs. ++ * Therefore, when an eBPF program using this helper is attached, ++ * a warning including PID and process name is printed to kernel ++ * logs. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_current_task_under_cgroup(struct bpf_map *map, u32 index) ++ * Description ++ * Check whether the probe is being run is the context of a given ++ * subset of the cgroup2 hierarchy. The cgroup2 to test is held by ++ * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. ++ * Return ++ * The return value depends on the result of the test, and can be: ++ * ++ * * 0, if current task belongs to the cgroup2. ++ * * 1, if current task does not belong to the cgroup2. ++ * * A negative error code, if an error occurred. ++ * ++ * int bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags) ++ * Description ++ * Resize (trim or grow) the packet associated to *skb* to the ++ * new *len*. The *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * The basic idea is that the helper performs the needed work to ++ * change the size of the packet, then the eBPF program rewrites ++ * the rest via helpers like **bpf_skb_store_bytes**\ (), ++ * **bpf_l3_csum_replace**\ (), **bpf_l3_csum_replace**\ () ++ * and others. This helper is a slow path utility intended for ++ * replies with control messages. And because it is targeted for ++ * slow path, the helper itself can afford to be slow: it ++ * implicitly linearizes, unclones and drops offloads from the ++ * *skb*. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_pull_data(struct sk_buff *skb, u32 len) ++ * Description ++ * Pull in non-linear data in case the *skb* is non-linear and not ++ * all of *len* are part of the linear section. Make *len* bytes ++ * from *skb* readable and writable. If a zero value is passed for ++ * *len*, then the whole length of the *skb* is pulled. ++ * ++ * This helper is only needed for reading and writing with direct ++ * packet access. ++ * ++ * For direct packet access, testing that offsets to access ++ * are within packet boundaries (test on *skb*\ **->data_end**) is ++ * susceptible to fail if offsets are invalid, or if the requested ++ * data is in non-linear parts of the *skb*. On failure the ++ * program can just bail out, or in the case of a non-linear ++ * buffer, use a helper to make the data available. The ++ * **bpf_skb_load_bytes**\ () helper is a first solution to access ++ * the data. Another one consists in using **bpf_skb_pull_data** ++ * to pull in once the non-linear parts, then retesting and ++ * eventually access the data. ++ * ++ * At the same time, this also makes sure the *skb* is uncloned, ++ * which is a necessary condition for direct write. As this needs ++ * to be an invariant for the write part only, the verifier ++ * detects writes and adds a prologue that is calling ++ * **bpf_skb_pull_data()** to effectively unclone the *skb* from ++ * the very beginning in case it is indeed cloned. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * s64 bpf_csum_update(struct sk_buff *skb, __wsum csum) ++ * Description ++ * Add the checksum *csum* into *skb*\ **->csum** in case the ++ * driver has supplied a checksum for the entire packet into that ++ * field. Return an error otherwise. This helper is intended to be ++ * used in combination with **bpf_csum_diff**\ (), in particular ++ * when the checksum needs to be updated after data has been ++ * written into the packet through direct packet access. ++ * Return ++ * The checksum on success, or a negative error code in case of ++ * failure. ++ * ++ * void bpf_set_hash_invalid(struct sk_buff *skb) ++ * Description ++ * Invalidate the current *skb*\ **->hash**. It can be used after ++ * mangling on headers through direct packet access, in order to ++ * indicate that the hash is outdated and to trigger a ++ * recalculation the next time the kernel tries to access this ++ * hash or when the **bpf_get_hash_recalc**\ () helper is called. ++ * ++ * int bpf_get_numa_node_id(void) ++ * Description ++ * Return the id of the current NUMA node. The primary use case ++ * for this helper is the selection of sockets for the local NUMA ++ * node, when the program is attached to sockets using the ++ * **SO_ATTACH_REUSEPORT_EBPF** option (see also **socket(7)**), ++ * but the helper is also available to other eBPF program types, ++ * similarly to **bpf_get_smp_processor_id**\ (). ++ * Return ++ * The id of current NUMA node. ++ * ++ * int bpf_skb_change_head(struct sk_buff *skb, u32 len, u64 flags) ++ * Description ++ * Grows headroom of packet associated to *skb* and adjusts the ++ * offset of the MAC header accordingly, adding *len* bytes of ++ * space. It automatically extends and reallocates memory as ++ * required. ++ * ++ * This helper can be used on a layer 3 *skb* to push a MAC header ++ * for redirection into a layer 2 device. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta) ++ * Description ++ * Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that ++ * it is possible to use a negative value for *delta*. This helper ++ * can be used to prepare the packet for pushing or popping ++ * headers. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_probe_read_str(void *dst, int size, const void *unsafe_ptr) ++ * Description ++ * Copy a NUL terminated string from an unsafe address ++ * *unsafe_ptr* to *dst*. The *size* should include the ++ * terminating NUL byte. In case the string length is smaller than ++ * *size*, the target is not padded with further NUL bytes. If the ++ * string length is larger than *size*, just *size*-1 bytes are ++ * copied and the last byte is set to NUL. ++ * ++ * On success, the length of the copied string is returned. This ++ * makes this helper useful in tracing programs for reading ++ * strings, and more importantly to get its length at runtime. See ++ * the following snippet: ++ * ++ * :: ++ * ++ * SEC("kprobe/sys_open") ++ * void bpf_sys_open(struct pt_regs *ctx) ++ * { ++ * char buf[PATHLEN]; // PATHLEN is defined to 256 ++ * int res = bpf_probe_read_str(buf, sizeof(buf), ++ * ctx->di); ++ * ++ * // Consume buf, for example push it to ++ * // userspace via bpf_perf_event_output(); we ++ * // can use res (the string length) as event ++ * // size, after checking its boundaries. ++ * } ++ * ++ * In comparison, using **bpf_probe_read()** helper here instead ++ * to read the string would require to estimate the length at ++ * compile time, and would often result in copying more memory ++ * than necessary. ++ * ++ * Another useful use case is when parsing individual process ++ * arguments or individual environment variables navigating ++ * *current*\ **->mm->arg_start** and *current*\ ++ * **->mm->env_start**: using this helper and the return value, ++ * one can quickly iterate at the right offset of the memory area. ++ * Return ++ * On success, the strictly positive length of the string, ++ * including the trailing NUL character. On error, a negative ++ * value. ++ * ++ * u64 bpf_get_socket_cookie(struct sk_buff *skb) ++ * Description ++ * If the **struct sk_buff** pointed by *skb* has a known socket, ++ * retrieve the cookie (generated by the kernel) of this socket. ++ * If no cookie has been set yet, generate a new cookie. Once ++ * generated, the socket cookie remains stable for the life of the ++ * socket. This helper can be useful for monitoring per socket ++ * networking traffic statistics as it provides a global socket ++ * identifier that can be assumed unique. ++ * Return ++ * A 8-byte long non-decreasing number on success, or 0 if the ++ * socket field is missing inside *skb*. ++ * ++ * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx) ++ * Description ++ * Equivalent to bpf_get_socket_cookie() helper that accepts ++ * *skb*, but gets socket from **struct bpf_sock_addr** context. ++ * Return ++ * A 8-byte long non-decreasing number. ++ * ++ * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx) ++ * Description ++ * Equivalent to bpf_get_socket_cookie() helper that accepts ++ * *skb*, but gets socket from **struct bpf_sock_ops** context. ++ * Return ++ * A 8-byte long non-decreasing number. ++ * ++ * u32 bpf_get_socket_uid(struct sk_buff *skb) ++ * Return ++ * The owner UID of the socket associated to *skb*. If the socket ++ * is **NULL**, or if it is not a full socket (i.e. if it is a ++ * time-wait or a request socket instead), **overflowuid** value ++ * is returned (note that **overflowuid** might also be the actual ++ * UID value for the socket). ++ * ++ * u32 bpf_set_hash(struct sk_buff *skb, u32 hash) ++ * Description ++ * Set the full hash for *skb* (set the field *skb*\ **->hash**) ++ * to value *hash*. ++ * Return ++ * 0 ++ * ++ * int bpf_setsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen) ++ * Description ++ * Emulate a call to **setsockopt()** on the socket associated to ++ * *bpf_socket*, which must be a full socket. The *level* at ++ * which the option resides and the name *optname* of the option ++ * must be specified, see **setsockopt(2)** for more information. ++ * The option value of length *optlen* is pointed by *optval*. ++ * ++ * This helper actually implements a subset of **setsockopt()**. ++ * It supports the following *level*\ s: ++ * ++ * * **SOL_SOCKET**, which supports the following *optname*\ s: ++ * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, ++ * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**. ++ * * **IPPROTO_TCP**, which supports the following *optname*\ s: ++ * **TCP_CONGESTION**, **TCP_BPF_IW**, ++ * **TCP_BPF_SNDCWND_CLAMP**. ++ * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. ++ * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_adjust_room(struct sk_buff *skb, s32 len_diff, u32 mode, u64 flags) ++ * Description ++ * Grow or shrink the room for data in the packet associated to ++ * *skb* by *len_diff*, and according to the selected *mode*. ++ * ++ * There are two supported modes at this time: ++ * ++ * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer ++ * (room space is added or removed below the layer 2 header). ++ * ++ * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer ++ * (room space is added or removed below the layer 3 header). ++ * ++ * The following flags are supported at this time: ++ * ++ * * **BPF_F_ADJ_ROOM_FIXED_GSO**: Do not adjust gso_size. ++ * Adjusting mss in this way is not allowed for datagrams. ++ * ++ * * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV4**, ++ * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV6**: ++ * Any new space is reserved to hold a tunnel header. ++ * Configure skb offsets and other fields accordingly. ++ * ++ * * **BPF_F_ADJ_ROOM_ENCAP_L4_GRE**, ++ * **BPF_F_ADJ_ROOM_ENCAP_L4_UDP**: ++ * Use with ENCAP_L3 flags to further specify the tunnel type. ++ * ++ * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): ++ * Use with ENCAP_L3/L4 flags to further specify the tunnel ++ * type; *len* is the length of the inner MAC header. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_redirect_map(struct bpf_map *map, u32 key, u64 flags) ++ * Description ++ * Redirect the packet to the endpoint referenced by *map* at ++ * index *key*. Depending on its type, this *map* can contain ++ * references to net devices (for forwarding packets through other ++ * ports), or to CPUs (for redirecting XDP frames to another CPU; ++ * but this is only implemented for native XDP (with driver ++ * support) as of this writing). ++ * ++ * The lower two bits of *flags* are used as the return code if ++ * the map lookup fails. This is so that the return value can be ++ * one of the XDP program return codes up to XDP_TX, as chosen by ++ * the caller. Any higher bits in the *flags* argument must be ++ * unset. ++ * ++ * When used to redirect packets to net devices, this helper ++ * provides a high performance increase over **bpf_redirect**\ (). ++ * This is due to various implementation details of the underlying ++ * mechanisms, one of which is the fact that **bpf_redirect_map**\ ++ * () tries to send packet as a "bulk" to the device. ++ * Return ++ * **XDP_REDIRECT** on success, or **XDP_ABORTED** on error. ++ * ++ * int bpf_sk_redirect_map(struct bpf_map *map, u32 key, u64 flags) ++ * Description ++ * Redirect the packet to the socket referenced by *map* (of type ++ * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and ++ * egress interfaces can be used for redirection. The ++ * **BPF_F_INGRESS** value in *flags* is used to make the ++ * distinction (ingress path is selected if the flag is present, ++ * egress path otherwise). This is the only flag supported for now. ++ * Return ++ * **SK_PASS** on success, or **SK_DROP** on error. ++ * ++ * int bpf_sock_map_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags) ++ * Description ++ * Add an entry to, or update a *map* referencing sockets. The ++ * *skops* is used as a new value for the entry associated to ++ * *key*. *flags* is one of: ++ * ++ * **BPF_NOEXIST** ++ * The entry for *key* must not exist in the map. ++ * **BPF_EXIST** ++ * The entry for *key* must already exist in the map. ++ * **BPF_ANY** ++ * No condition on the existence of the entry for *key*. ++ * ++ * If the *map* has eBPF programs (parser and verdict), those will ++ * be inherited by the socket being added. If the socket is ++ * already attached to eBPF programs, this results in an error. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta) ++ * Description ++ * Adjust the address pointed by *xdp_md*\ **->data_meta** by ++ * *delta* (which can be positive or negative). Note that this ++ * operation modifies the address stored in *xdp_md*\ **->data**, ++ * so the latter must be loaded only after the helper has been ++ * called. ++ * ++ * The use of *xdp_md*\ **->data_meta** is optional and programs ++ * are not required to use it. The rationale is that when the ++ * packet is processed with XDP (e.g. as DoS filter), it is ++ * possible to push further meta data along with it before passing ++ * to the stack, and to give the guarantee that an ingress eBPF ++ * program attached as a TC classifier on the same device can pick ++ * this up for further post-processing. Since TC works with socket ++ * buffers, it remains possible to set from XDP the **mark** or ++ * **priority** pointers, or other pointers for the socket buffer. ++ * Having this scratch space generic and programmable allows for ++ * more flexibility as the user is free to store whatever meta ++ * data they need. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_perf_event_read_value(struct bpf_map *map, u64 flags, struct bpf_perf_event_value *buf, u32 buf_size) ++ * Description ++ * Read the value of a perf event counter, and store it into *buf* ++ * of size *buf_size*. This helper relies on a *map* of type ++ * **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of the perf event ++ * counter is selected when *map* is updated with perf event file ++ * descriptors. The *map* is an array whose size is the number of ++ * available CPUs, and each cell contains a value relative to one ++ * CPU. The value to retrieve is indicated by *flags*, that ++ * contains the index of the CPU to look up, masked with ++ * **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to ++ * **BPF_F_CURRENT_CPU** to indicate that the value for the ++ * current CPU should be retrieved. ++ * ++ * This helper behaves in a way close to ++ * **bpf_perf_event_read**\ () helper, save that instead of ++ * just returning the value observed, it fills the *buf* ++ * structure. This allows for additional data to be retrieved: in ++ * particular, the enabled and running times (in *buf*\ ++ * **->enabled** and *buf*\ **->running**, respectively) are ++ * copied. In general, **bpf_perf_event_read_value**\ () is ++ * recommended over **bpf_perf_event_read**\ (), which has some ++ * ABI issues and provides fewer functionalities. ++ * ++ * These values are interesting, because hardware PMU (Performance ++ * Monitoring Unit) counters are limited resources. When there are ++ * more PMU based perf events opened than available counters, ++ * kernel will multiplex these events so each event gets certain ++ * percentage (but not all) of the PMU time. In case that ++ * multiplexing happens, the number of samples or counter value ++ * will not reflect the case compared to when no multiplexing ++ * occurs. This makes comparison between different runs difficult. ++ * Typically, the counter value should be normalized before ++ * comparing to other experiments. The usual normalization is done ++ * as follows. ++ * ++ * :: ++ * ++ * normalized_counter = counter * t_enabled / t_running ++ * ++ * Where t_enabled is the time enabled for event and t_running is ++ * the time running for event since last normalization. The ++ * enabled and running times are accumulated since the perf event ++ * open. To achieve scaling factor between two invocations of an ++ * eBPF program, users can can use CPU id as the key (which is ++ * typical for perf array usage model) to remember the previous ++ * value and do the calculation inside the eBPF program. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_perf_prog_read_value(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, u32 buf_size) ++ * Description ++ * For en eBPF program attached to a perf event, retrieve the ++ * value of the event counter associated to *ctx* and store it in ++ * the structure pointed by *buf* and of size *buf_size*. Enabled ++ * and running times are also stored in the structure (see ++ * description of helper **bpf_perf_event_read_value**\ () for ++ * more details). ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_getsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen) ++ * Description ++ * Emulate a call to **getsockopt()** on the socket associated to ++ * *bpf_socket*, which must be a full socket. The *level* at ++ * which the option resides and the name *optname* of the option ++ * must be specified, see **getsockopt(2)** for more information. ++ * The retrieved value is stored in the structure pointed by ++ * *opval* and of length *optlen*. ++ * ++ * This helper actually implements a subset of **getsockopt()**. ++ * It supports the following *level*\ s: ++ * ++ * * **IPPROTO_TCP**, which supports *optname* ++ * **TCP_CONGESTION**. ++ * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. ++ * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_override_return(struct pt_regs *regs, u64 rc) ++ * Description ++ * Used for error injection, this helper uses kprobes to override ++ * the return value of the probed function, and to set it to *rc*. ++ * The first argument is the context *regs* on which the kprobe ++ * works. ++ * ++ * This helper works by setting setting the PC (program counter) ++ * to an override function which is run in place of the original ++ * probed function. This means the probed function is not run at ++ * all. The replacement function just returns with the required ++ * value. ++ * ++ * This helper has security implications, and thus is subject to ++ * restrictions. It is only available if the kernel was compiled ++ * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration ++ * option, and in this case it only works on functions tagged with ++ * **ALLOW_ERROR_INJECTION** in the kernel code. ++ * ++ * Also, the helper is only available for the architectures having ++ * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing, ++ * x86 architecture is the only one to support this feature. ++ * Return ++ * 0 ++ * ++ * int bpf_sock_ops_cb_flags_set(struct bpf_sock_ops *bpf_sock, int argval) ++ * Description ++ * Attempt to set the value of the **bpf_sock_ops_cb_flags** field ++ * for the full TCP socket associated to *bpf_sock_ops* to ++ * *argval*. ++ * ++ * The primary use of this field is to determine if there should ++ * be calls to eBPF programs of type ++ * **BPF_PROG_TYPE_SOCK_OPS** at various points in the TCP ++ * code. A program of the same type can change its value, per ++ * connection and as necessary, when the connection is ++ * established. This field is directly accessible for reading, but ++ * this helper must be used for updates in order to return an ++ * error if an eBPF program tries to set a callback that is not ++ * supported in the current kernel. ++ * ++ * *argval* is a flag array which can combine these flags: ++ * ++ * * **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out) ++ * * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission) ++ * * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change) ++ * * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT) ++ * ++ * Therefore, this function can be used to clear a callback flag by ++ * setting the appropriate bit to zero. e.g. to disable the RTO ++ * callback: ++ * ++ * **bpf_sock_ops_cb_flags_set(bpf_sock,** ++ * **bpf_sock->bpf_sock_ops_cb_flags & ~BPF_SOCK_OPS_RTO_CB_FLAG)** ++ * ++ * Here are some examples of where one could call such eBPF ++ * program: ++ * ++ * * When RTO fires. ++ * * When a packet is retransmitted. ++ * * When the connection terminates. ++ * * When a packet is sent. ++ * * When a packet is received. ++ * Return ++ * Code **-EINVAL** if the socket is not a full TCP socket; ++ * otherwise, a positive number containing the bits that could not ++ * be set is returned (which comes down to 0 if all bits were set ++ * as required). ++ * ++ * int bpf_msg_redirect_map(struct sk_msg_buff *msg, struct bpf_map *map, u32 key, u64 flags) ++ * Description ++ * This helper is used in programs implementing policies at the ++ * socket level. If the message *msg* is allowed to pass (i.e. if ++ * the verdict eBPF program returns **SK_PASS**), redirect it to ++ * the socket referenced by *map* (of type ++ * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and ++ * egress interfaces can be used for redirection. The ++ * **BPF_F_INGRESS** value in *flags* is used to make the ++ * distinction (ingress path is selected if the flag is present, ++ * egress path otherwise). This is the only flag supported for now. ++ * Return ++ * **SK_PASS** on success, or **SK_DROP** on error. ++ * ++ * int bpf_msg_apply_bytes(struct sk_msg_buff *msg, u32 bytes) ++ * Description ++ * For socket policies, apply the verdict of the eBPF program to ++ * the next *bytes* (number of bytes) of message *msg*. ++ * ++ * For example, this helper can be used in the following cases: ++ * ++ * * A single **sendmsg**\ () or **sendfile**\ () system call ++ * contains multiple logical messages that the eBPF program is ++ * supposed to read and for which it should apply a verdict. ++ * * An eBPF program only cares to read the first *bytes* of a ++ * *msg*. If the message has a large payload, then setting up ++ * and calling the eBPF program repeatedly for all bytes, even ++ * though the verdict is already known, would create unnecessary ++ * overhead. ++ * ++ * When called from within an eBPF program, the helper sets a ++ * counter internal to the BPF infrastructure, that is used to ++ * apply the last verdict to the next *bytes*. If *bytes* is ++ * smaller than the current data being processed from a ++ * **sendmsg**\ () or **sendfile**\ () system call, the first ++ * *bytes* will be sent and the eBPF program will be re-run with ++ * the pointer for start of data pointing to byte number *bytes* ++ * **+ 1**. If *bytes* is larger than the current data being ++ * processed, then the eBPF verdict will be applied to multiple ++ * **sendmsg**\ () or **sendfile**\ () calls until *bytes* are ++ * consumed. ++ * ++ * Note that if a socket closes with the internal counter holding ++ * a non-zero value, this is not a problem because data is not ++ * being buffered for *bytes* and is sent as it is received. ++ * Return ++ * 0 ++ * ++ * int bpf_msg_cork_bytes(struct sk_msg_buff *msg, u32 bytes) ++ * Description ++ * For socket policies, prevent the execution of the verdict eBPF ++ * program for message *msg* until *bytes* (byte number) have been ++ * accumulated. ++ * ++ * This can be used when one needs a specific number of bytes ++ * before a verdict can be assigned, even if the data spans ++ * multiple **sendmsg**\ () or **sendfile**\ () calls. The extreme ++ * case would be a user calling **sendmsg**\ () repeatedly with ++ * 1-byte long message segments. Obviously, this is bad for ++ * performance, but it is still valid. If the eBPF program needs ++ * *bytes* bytes to validate a header, this helper can be used to ++ * prevent the eBPF program to be called again until *bytes* have ++ * been accumulated. ++ * Return ++ * 0 ++ * ++ * int bpf_msg_pull_data(struct sk_msg_buff *msg, u32 start, u32 end, u64 flags) ++ * Description ++ * For socket policies, pull in non-linear data from user space ++ * for *msg* and set pointers *msg*\ **->data** and *msg*\ ++ * **->data_end** to *start* and *end* bytes offsets into *msg*, ++ * respectively. ++ * ++ * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a ++ * *msg* it can only parse data that the (**data**, **data_end**) ++ * pointers have already consumed. For **sendmsg**\ () hooks this ++ * is likely the first scatterlist element. But for calls relying ++ * on the **sendpage** handler (e.g. **sendfile**\ ()) this will ++ * be the range (**0**, **0**) because the data is shared with ++ * user space and by default the objective is to avoid allowing ++ * user space to modify data while (or after) eBPF verdict is ++ * being decided. This helper can be used to pull in data and to ++ * set the start and end pointer to given values. Data will be ++ * copied if necessary (i.e. if data was not linear and if start ++ * and end pointers do not point to the same chunk). ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_bind(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) ++ * Description ++ * Bind the socket associated to *ctx* to the address pointed by ++ * *addr*, of length *addr_len*. This allows for making outgoing ++ * connection from the desired IP address, which can be useful for ++ * example when all processes inside a cgroup should use one ++ * single IP address on a host that has multiple IP configured. ++ * ++ * This helper works for IPv4 and IPv6, TCP and UDP sockets. The ++ * domain (*addr*\ **->sa_family**) must be **AF_INET** (or ++ * **AF_INET6**). Looking for a free port to bind to can be ++ * expensive, therefore binding to port is not permitted by the ++ * helper: *addr*\ **->sin_port** (or **sin6_port**, respectively) ++ * must be set to zero. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_xdp_adjust_tail(struct xdp_buff *xdp_md, int delta) ++ * Description ++ * Adjust (move) *xdp_md*\ **->data_end** by *delta* bytes. It is ++ * only possible to shrink the packet as of this writing, ++ * therefore *delta* must be a negative integer. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_skb_get_xfrm_state(struct sk_buff *skb, u32 index, struct bpf_xfrm_state *xfrm_state, u32 size, u64 flags) ++ * Description ++ * Retrieve the XFRM state (IP transform framework, see also ++ * **ip-xfrm(8)**) at *index* in XFRM "security path" for *skb*. ++ * ++ * The retrieved value is stored in the **struct bpf_xfrm_state** ++ * pointed by *xfrm_state* and of length *size*. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * This helper is available only if the kernel was compiled with ++ * **CONFIG_XFRM** configuration option. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_get_stack(struct pt_regs *regs, void *buf, u32 size, u64 flags) ++ * Description ++ * Return a user or a kernel stack in bpf program provided buffer. ++ * To achieve this, the helper needs *ctx*, which is a pointer ++ * to the context on which the tracing program is executed. ++ * To store the stacktrace, the bpf program provides *buf* with ++ * a nonnegative *size*. ++ * ++ * The last argument, *flags*, holds the number of stack frames to ++ * skip (from 0 to 255), masked with ++ * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set ++ * the following flags: ++ * ++ * **BPF_F_USER_STACK** ++ * Collect a user space stack instead of a kernel stack. ++ * **BPF_F_USER_BUILD_ID** ++ * Collect buildid+offset instead of ips for user stack, ++ * only valid if **BPF_F_USER_STACK** is also specified. ++ * ++ * **bpf_get_stack**\ () can collect up to ++ * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject ++ * to sufficient large buffer size. Note that ++ * this limit can be controlled with the **sysctl** program, and ++ * that it should be manually increased in order to profile long ++ * user stacks (such as stacks for Java programs). To do so, use: ++ * ++ * :: ++ * ++ * # sysctl kernel.perf_event_max_stack= ++ * Return ++ * A non-negative value equal to or less than *size* on success, ++ * or a negative error in case of failure. ++ * ++ * int bpf_skb_load_bytes_relative(const struct sk_buff *skb, u32 offset, void *to, u32 len, u32 start_header) ++ * Description ++ * This helper is similar to **bpf_skb_load_bytes**\ () in that ++ * it provides an easy way to load *len* bytes from *offset* ++ * from the packet associated to *skb*, into the buffer pointed ++ * by *to*. The difference to **bpf_skb_load_bytes**\ () is that ++ * a fifth argument *start_header* exists in order to select a ++ * base offset to start from. *start_header* can be one of: ++ * ++ * **BPF_HDR_START_MAC** ++ * Base offset to load data from is *skb*'s mac header. ++ * **BPF_HDR_START_NET** ++ * Base offset to load data from is *skb*'s network header. ++ * ++ * In general, "direct packet access" is the preferred method to ++ * access packet data, however, this helper is in particular useful ++ * in socket filters where *skb*\ **->data** does not always point ++ * to the start of the mac header and where "direct packet access" ++ * is not available. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_fib_lookup(void *ctx, struct bpf_fib_lookup *params, int plen, u32 flags) ++ * Description ++ * Do FIB lookup in kernel tables using parameters in *params*. ++ * If lookup is successful and result shows packet is to be ++ * forwarded, the neighbor tables are searched for the nexthop. ++ * If successful (ie., FIB lookup shows forwarding and nexthop ++ * is resolved), the nexthop address is returned in ipv4_dst ++ * or ipv6_dst based on family, smac is set to mac address of ++ * egress device, dmac is set to nexthop mac address, rt_metric ++ * is set to metric from route (IPv4/IPv6 only), and ifindex ++ * is set to the device index of the nexthop from the FIB lookup. ++ * ++ * *plen* argument is the size of the passed in struct. ++ * *flags* argument can be a combination of one or more of the ++ * following values: ++ * ++ * **BPF_FIB_LOOKUP_DIRECT** ++ * Do a direct table lookup vs full lookup using FIB ++ * rules. ++ * **BPF_FIB_LOOKUP_OUTPUT** ++ * Perform lookup from an egress perspective (default is ++ * ingress). ++ * ++ * *ctx* is either **struct xdp_md** for XDP programs or ++ * **struct sk_buff** tc cls_act programs. ++ * Return ++ * * < 0 if any input argument is invalid ++ * * 0 on success (packet is forwarded, nexthop neighbor exists) ++ * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the ++ * packet is not forwarded or needs assist from full stack ++ * ++ * int bpf_sock_hash_update(struct bpf_sock_ops_kern *skops, struct bpf_map *map, void *key, u64 flags) ++ * Description ++ * Add an entry to, or update a sockhash *map* referencing sockets. ++ * The *skops* is used as a new value for the entry associated to ++ * *key*. *flags* is one of: ++ * ++ * **BPF_NOEXIST** ++ * The entry for *key* must not exist in the map. ++ * **BPF_EXIST** ++ * The entry for *key* must already exist in the map. ++ * **BPF_ANY** ++ * No condition on the existence of the entry for *key*. ++ * ++ * If the *map* has eBPF programs (parser and verdict), those will ++ * be inherited by the socket being added. If the socket is ++ * already attached to eBPF programs, this results in an error. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_msg_redirect_hash(struct sk_msg_buff *msg, struct bpf_map *map, void *key, u64 flags) ++ * Description ++ * This helper is used in programs implementing policies at the ++ * socket level. If the message *msg* is allowed to pass (i.e. if ++ * the verdict eBPF program returns **SK_PASS**), redirect it to ++ * the socket referenced by *map* (of type ++ * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and ++ * egress interfaces can be used for redirection. The ++ * **BPF_F_INGRESS** value in *flags* is used to make the ++ * distinction (ingress path is selected if the flag is present, ++ * egress path otherwise). This is the only flag supported for now. ++ * Return ++ * **SK_PASS** on success, or **SK_DROP** on error. ++ * ++ * int bpf_sk_redirect_hash(struct sk_buff *skb, struct bpf_map *map, void *key, u64 flags) ++ * Description ++ * This helper is used in programs implementing policies at the ++ * skb socket level. If the sk_buff *skb* is allowed to pass (i.e. ++ * if the verdeict eBPF program returns **SK_PASS**), redirect it ++ * to the socket referenced by *map* (of type ++ * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and ++ * egress interfaces can be used for redirection. The ++ * **BPF_F_INGRESS** value in *flags* is used to make the ++ * distinction (ingress path is selected if the flag is present, ++ * egress otherwise). This is the only flag supported for now. ++ * Return ++ * **SK_PASS** on success, or **SK_DROP** on error. ++ * ++ * int bpf_lwt_push_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len) ++ * Description ++ * Encapsulate the packet associated to *skb* within a Layer 3 ++ * protocol header. This header is provided in the buffer at ++ * address *hdr*, with *len* its size in bytes. *type* indicates ++ * the protocol of the header and can be one of: ++ * ++ * **BPF_LWT_ENCAP_SEG6** ++ * IPv6 encapsulation with Segment Routing Header ++ * (**struct ipv6_sr_hdr**). *hdr* only contains the SRH, ++ * the IPv6 header is computed by the kernel. ++ * **BPF_LWT_ENCAP_SEG6_INLINE** ++ * Only works if *skb* contains an IPv6 packet. Insert a ++ * Segment Routing Header (**struct ipv6_sr_hdr**) inside ++ * the IPv6 header. ++ * **BPF_LWT_ENCAP_IP** ++ * IP encapsulation (GRE/GUE/IPIP/etc). The outer header ++ * must be IPv4 or IPv6, followed by zero or more ++ * additional headers, up to **LWT_BPF_MAX_HEADROOM** ++ * total bytes in all prepended headers. Please note that ++ * if **skb_is_gso**\ (*skb*) is true, no more than two ++ * headers can be prepended, and the inner header, if ++ * present, should be either GRE or UDP/GUE. ++ * ++ * **BPF_LWT_ENCAP_SEG6**\ \* types can be called by BPF programs ++ * of type **BPF_PROG_TYPE_LWT_IN**; **BPF_LWT_ENCAP_IP** type can ++ * be called by bpf programs of types **BPF_PROG_TYPE_LWT_IN** and ++ * **BPF_PROG_TYPE_LWT_XMIT**. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_lwt_seg6_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len) ++ * Description ++ * Store *len* bytes from address *from* into the packet ++ * associated to *skb*, at *offset*. Only the flags, tag and TLVs ++ * inside the outermost IPv6 Segment Routing Header can be ++ * modified through this helper. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_lwt_seg6_adjust_srh(struct sk_buff *skb, u32 offset, s32 delta) ++ * Description ++ * Adjust the size allocated to TLVs in the outermost IPv6 ++ * Segment Routing Header contained in the packet associated to ++ * *skb*, at position *offset* by *delta* bytes. Only offsets ++ * after the segments are accepted. *delta* can be as well ++ * positive (growing) as negative (shrinking). ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_lwt_seg6_action(struct sk_buff *skb, u32 action, void *param, u32 param_len) ++ * Description ++ * Apply an IPv6 Segment Routing action of type *action* to the ++ * packet associated to *skb*. Each action takes a parameter ++ * contained at address *param*, and of length *param_len* bytes. ++ * *action* can be one of: ++ * ++ * **SEG6_LOCAL_ACTION_END_X** ++ * End.X action: Endpoint with Layer-3 cross-connect. ++ * Type of *param*: **struct in6_addr**. ++ * **SEG6_LOCAL_ACTION_END_T** ++ * End.T action: Endpoint with specific IPv6 table lookup. ++ * Type of *param*: **int**. ++ * **SEG6_LOCAL_ACTION_END_B6** ++ * End.B6 action: Endpoint bound to an SRv6 policy. ++ * Type of *param*: **struct ipv6_sr_hdr**. ++ * **SEG6_LOCAL_ACTION_END_B6_ENCAP** ++ * End.B6.Encap action: Endpoint bound to an SRv6 ++ * encapsulation policy. ++ * Type of *param*: **struct ipv6_sr_hdr**. ++ * ++ * A call to this helper is susceptible to change the underlying ++ * packet buffer. Therefore, at load time, all checks on pointers ++ * previously done by the verifier are invalidated and must be ++ * performed again, if the helper is used in combination with ++ * direct packet access. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_rc_repeat(void *ctx) ++ * Description ++ * This helper is used in programs implementing IR decoding, to ++ * report a successfully decoded repeat key message. This delays ++ * the generation of a key up event for previously generated ++ * key down event. ++ * ++ * Some IR protocols like NEC have a special IR message for ++ * repeating last button, for when a button is held down. ++ * ++ * The *ctx* should point to the lirc sample as passed into ++ * the program. ++ * ++ * This helper is only available is the kernel was compiled with ++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to ++ * "**y**". ++ * Return ++ * 0 ++ * ++ * int bpf_rc_keydown(void *ctx, u32 protocol, u64 scancode, u32 toggle) ++ * Description ++ * This helper is used in programs implementing IR decoding, to ++ * report a successfully decoded key press with *scancode*, ++ * *toggle* value in the given *protocol*. The scancode will be ++ * translated to a keycode using the rc keymap, and reported as ++ * an input key down event. After a period a key up event is ++ * generated. This period can be extended by calling either ++ * **bpf_rc_keydown**\ () again with the same values, or calling ++ * **bpf_rc_repeat**\ (). ++ * ++ * Some protocols include a toggle bit, in case the button was ++ * released and pressed again between consecutive scancodes. ++ * ++ * The *ctx* should point to the lirc sample as passed into ++ * the program. ++ * ++ * The *protocol* is the decoded protocol number (see ++ * **enum rc_proto** for some predefined values). ++ * ++ * This helper is only available is the kernel was compiled with ++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to ++ * "**y**". ++ * Return ++ * 0 ++ * ++ * u64 bpf_skb_cgroup_id(struct sk_buff *skb) ++ * Description ++ * Return the cgroup v2 id of the socket associated with the *skb*. ++ * This is roughly similar to the **bpf_get_cgroup_classid**\ () ++ * helper for cgroup v1 by providing a tag resp. identifier that ++ * can be matched on or used for map lookups e.g. to implement ++ * policy. The cgroup v2 id of a given path in the hierarchy is ++ * exposed in user space through the f_handle API in order to get ++ * to the same 64-bit id. ++ * ++ * This helper can be used on TC egress path, but not on ingress, ++ * and is available only if the kernel was compiled with the ++ * **CONFIG_SOCK_CGROUP_DATA** configuration option. ++ * Return ++ * The id is returned or 0 in case the id could not be retrieved. ++ * ++ * u64 bpf_get_current_cgroup_id(void) ++ * Return ++ * A 64-bit integer containing the current cgroup id based ++ * on the cgroup within which the current task is running. ++ * ++ * void *bpf_get_local_storage(void *map, u64 flags) ++ * Description ++ * Get the pointer to the local storage area. ++ * The type and the size of the local storage is defined ++ * by the *map* argument. ++ * The *flags* meaning is specific for each map type, ++ * and has to be 0 for cgroup local storage. ++ * ++ * Depending on the BPF program type, a local storage area ++ * can be shared between multiple instances of the BPF program, ++ * running simultaneously. ++ * ++ * A user should care about the synchronization by himself. ++ * For example, by using the **BPF_STX_XADD** instruction to alter ++ * the shared data. ++ * Return ++ * A pointer to the local storage area. ++ * ++ * int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) ++ * Description ++ * Select a **SO_REUSEPORT** socket from a ++ * **BPF_MAP_TYPE_REUSEPORT_ARRAY** *map*. ++ * It checks the selected socket is matching the incoming ++ * request in the socket buffer. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * u64 bpf_skb_ancestor_cgroup_id(struct sk_buff *skb, int ancestor_level) ++ * Description ++ * Return id of cgroup v2 that is ancestor of cgroup associated ++ * with the *skb* at the *ancestor_level*. The root cgroup is at ++ * *ancestor_level* zero and each step down the hierarchy ++ * increments the level. If *ancestor_level* == level of cgroup ++ * associated with *skb*, then return value will be same as that ++ * of **bpf_skb_cgroup_id**\ (). ++ * ++ * The helper is useful to implement policies based on cgroups ++ * that are upper in hierarchy than immediate cgroup associated ++ * with *skb*. ++ * ++ * The format of returned id and helper limitations are same as in ++ * **bpf_skb_cgroup_id**\ (). ++ * Return ++ * The id is returned or 0 in case the id could not be retrieved. ++ * ++ * struct bpf_sock *bpf_sk_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) ++ * Description ++ * Look for TCP socket matching *tuple*, optionally in a child ++ * network namespace *netns*. The return value must be checked, ++ * and if non-**NULL**, released via **bpf_sk_release**\ (). ++ * ++ * The *ctx* should point to the context of the program, such as ++ * the skb or socket (depending on the hook in use). This is used ++ * to determine the base network namespace for the lookup. ++ * ++ * *tuple_size* must be one of: ++ * ++ * **sizeof**\ (*tuple*\ **->ipv4**) ++ * Look for an IPv4 socket. ++ * **sizeof**\ (*tuple*\ **->ipv6**) ++ * Look for an IPv6 socket. ++ * ++ * If the *netns* is a negative signed 32-bit integer, then the ++ * socket lookup table in the netns associated with the *ctx* will ++ * will be used. For the TC hooks, this is the netns of the device ++ * in the skb. For socket hooks, this is the netns of the socket. ++ * If *netns* is any other signed 32-bit value greater than or ++ * equal to zero then it specifies the ID of the netns relative to ++ * the netns associated with the *ctx*. *netns* values beyond the ++ * range of 32-bit integers are reserved for future use. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * This helper is available only if the kernel was compiled with ++ * **CONFIG_NET** configuration option. ++ * Return ++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. ++ * For sockets with reuseport option, the **struct bpf_sock** ++ * result is from *reuse*\ **->socks**\ [] using the hash of the ++ * tuple. ++ * ++ * struct bpf_sock *bpf_sk_lookup_udp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) ++ * Description ++ * Look for UDP socket matching *tuple*, optionally in a child ++ * network namespace *netns*. The return value must be checked, ++ * and if non-**NULL**, released via **bpf_sk_release**\ (). ++ * ++ * The *ctx* should point to the context of the program, such as ++ * the skb or socket (depending on the hook in use). This is used ++ * to determine the base network namespace for the lookup. ++ * ++ * *tuple_size* must be one of: ++ * ++ * **sizeof**\ (*tuple*\ **->ipv4**) ++ * Look for an IPv4 socket. ++ * **sizeof**\ (*tuple*\ **->ipv6**) ++ * Look for an IPv6 socket. ++ * ++ * If the *netns* is a negative signed 32-bit integer, then the ++ * socket lookup table in the netns associated with the *ctx* will ++ * will be used. For the TC hooks, this is the netns of the device ++ * in the skb. For socket hooks, this is the netns of the socket. ++ * If *netns* is any other signed 32-bit value greater than or ++ * equal to zero then it specifies the ID of the netns relative to ++ * the netns associated with the *ctx*. *netns* values beyond the ++ * range of 32-bit integers are reserved for future use. ++ * ++ * All values for *flags* are reserved for future usage, and must ++ * be left at zero. ++ * ++ * This helper is available only if the kernel was compiled with ++ * **CONFIG_NET** configuration option. ++ * Return ++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. ++ * For sockets with reuseport option, the **struct bpf_sock** ++ * result is from *reuse*\ **->socks**\ [] using the hash of the ++ * tuple. ++ * ++ * int bpf_sk_release(struct bpf_sock *sock) ++ * Description ++ * Release the reference held by *sock*. *sock* must be a ++ * non-**NULL** pointer that was returned from ++ * **bpf_sk_lookup_xxx**\ (). ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_map_push_elem(struct bpf_map *map, const void *value, u64 flags) ++ * Description ++ * Push an element *value* in *map*. *flags* is one of: ++ * ++ * **BPF_EXIST** ++ * If the queue/stack is full, the oldest element is ++ * removed to make room for this. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_map_pop_elem(struct bpf_map *map, void *value) ++ * Description ++ * Pop an element from *map*. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_map_peek_elem(struct bpf_map *map, void *value) ++ * Description ++ * Get an element from *map* without removing it. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_msg_push_data(struct sk_buff *skb, u32 start, u32 len, u64 flags) ++ * Description ++ * For socket policies, insert *len* bytes into *msg* at offset ++ * *start*. ++ * ++ * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a ++ * *msg* it may want to insert metadata or options into the *msg*. ++ * This can later be read and used by any of the lower layer BPF ++ * hooks. ++ * ++ * This helper may fail if under memory pressure (a malloc ++ * fails) in these cases BPF programs will get an appropriate ++ * error and BPF programs will need to handle them. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags) ++ * Description ++ * Will remove *pop* bytes from a *msg* starting at byte *start*. ++ * This may result in **ENOMEM** errors under certain situations if ++ * an allocation and copy are required due to a full ring buffer. ++ * However, the helper will try to avoid doing the allocation ++ * if possible. Other errors can occur if input parameters are ++ * invalid either due to *start* byte not being valid part of *msg* ++ * payload and/or *pop* value being to large. ++ * Return ++ * 0 on success, or a negative error in case of failure. ++ * ++ * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y) ++ * Description ++ * This helper is used in programs implementing IR decoding, to ++ * report a successfully decoded pointer movement. ++ * ++ * The *ctx* should point to the lirc sample as passed into ++ * the program. ++ * ++ * This helper is only available is the kernel was compiled with ++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to ++ * "**y**". ++ * Return ++ * 0 ++ * ++ * int bpf_spin_lock(struct bpf_spin_lock *lock) ++ * Description ++ * Acquire a spinlock represented by the pointer *lock*, which is ++ * stored as part of a value of a map. Taking the lock allows to ++ * safely update the rest of the fields in that value. The ++ * spinlock can (and must) later be released with a call to ++ * **bpf_spin_unlock**\ (\ *lock*\ ). ++ * ++ * Spinlocks in BPF programs come with a number of restrictions ++ * and constraints: ++ * ++ * * **bpf_spin_lock** objects are only allowed inside maps of ++ * types **BPF_MAP_TYPE_HASH** and **BPF_MAP_TYPE_ARRAY** (this ++ * list could be extended in the future). ++ * * BTF description of the map is mandatory. ++ * * The BPF program can take ONE lock at a time, since taking two ++ * or more could cause dead locks. ++ * * Only one **struct bpf_spin_lock** is allowed per map element. ++ * * When the lock is taken, calls (either BPF to BPF or helpers) ++ * are not allowed. ++ * * The **BPF_LD_ABS** and **BPF_LD_IND** instructions are not ++ * allowed inside a spinlock-ed region. ++ * * The BPF program MUST call **bpf_spin_unlock**\ () to release ++ * the lock, on all execution paths, before it returns. ++ * * The BPF program can access **struct bpf_spin_lock** only via ++ * the **bpf_spin_lock**\ () and **bpf_spin_unlock**\ () ++ * helpers. Loading or storing data into the **struct ++ * bpf_spin_lock** *lock*\ **;** field of a map is not allowed. ++ * * To use the **bpf_spin_lock**\ () helper, the BTF description ++ * of the map value must be a struct and have **struct ++ * bpf_spin_lock** *anyname*\ **;** field at the top level. ++ * Nested lock inside another struct is not allowed. ++ * * The **struct bpf_spin_lock** *lock* field in a map value must ++ * be aligned on a multiple of 4 bytes in that value. ++ * * Syscall with command **BPF_MAP_LOOKUP_ELEM** does not copy ++ * the **bpf_spin_lock** field to user space. ++ * * Syscall with command **BPF_MAP_UPDATE_ELEM**, or update from ++ * a BPF program, do not update the **bpf_spin_lock** field. ++ * * **bpf_spin_lock** cannot be on the stack or inside a ++ * networking packet (it can only be inside of a map values). ++ * * **bpf_spin_lock** is available to root only. ++ * * Tracing programs and socket filter programs cannot use ++ * **bpf_spin_lock**\ () due to insufficient preemption checks ++ * (but this may change in the future). ++ * * **bpf_spin_lock** is not allowed in inner maps of map-in-map. ++ * Return ++ * 0 ++ * ++ * int bpf_spin_unlock(struct bpf_spin_lock *lock) ++ * Description ++ * Release the *lock* previously locked by a call to ++ * **bpf_spin_lock**\ (\ *lock*\ ). ++ * Return ++ * 0 ++ * ++ * struct bpf_sock *bpf_sk_fullsock(struct bpf_sock *sk) ++ * Description ++ * This helper gets a **struct bpf_sock** pointer such ++ * that all the fields in this **bpf_sock** can be accessed. ++ * Return ++ * A **struct bpf_sock** pointer on success, or **NULL** in ++ * case of failure. ++ * ++ * struct bpf_tcp_sock *bpf_tcp_sock(struct bpf_sock *sk) ++ * Description ++ * This helper gets a **struct bpf_tcp_sock** pointer from a ++ * **struct bpf_sock** pointer. ++ * Return ++ * A **struct bpf_tcp_sock** pointer on success, or **NULL** in ++ * case of failure. ++ * ++ * int bpf_skb_ecn_set_ce(struct sk_buf *skb) ++ * Description ++ * Set ECN (Explicit Congestion Notification) field of IP header ++ * to **CE** (Congestion Encountered) if current value is **ECT** ++ * (ECN Capable Transport). Otherwise, do nothing. Works with IPv6 ++ * and IPv4. ++ * Return ++ * 1 if the **CE** flag is set (either by the current helper call ++ * or because it was already present), 0 if it is not set. ++ * ++ * struct bpf_sock *bpf_get_listener_sock(struct bpf_sock *sk) ++ * Description ++ * Return a **struct bpf_sock** pointer in **TCP_LISTEN** state. ++ * **bpf_sk_release**\ () is unnecessary and not allowed. ++ * Return ++ * A **struct bpf_sock** pointer on success, or **NULL** in ++ * case of failure. ++ * ++ * struct bpf_sock *bpf_skc_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) ++ * Description ++ * Look for TCP socket matching *tuple*, optionally in a child ++ * network namespace *netns*. The return value must be checked, ++ * and if non-**NULL**, released via **bpf_sk_release**\ (). ++ * ++ * This function is identical to **bpf_sk_lookup_tcp**\ (), except ++ * that it also returns timewait or request sockets. Use ++ * **bpf_sk_fullsock**\ () or **bpf_tcp_sock**\ () to access the ++ * full structure. ++ * ++ * This helper is available only if the kernel was compiled with ++ * **CONFIG_NET** configuration option. ++ * Return ++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. ++ * For sockets with reuseport option, the **struct bpf_sock** ++ * result is from *reuse*\ **->socks**\ [] using the hash of the ++ * tuple. ++ * ++ * int bpf_tcp_check_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) ++ * Description ++ * Check whether *iph* and *th* contain a valid SYN cookie ACK for ++ * the listening socket in *sk*. ++ * ++ * *iph* points to the start of the IPv4 or IPv6 header, while ++ * *iph_len* contains **sizeof**\ (**struct iphdr**) or ++ * **sizeof**\ (**struct ip6hdr**). ++ * ++ * *th* points to the start of the TCP header, while *th_len* ++ * contains **sizeof**\ (**struct tcphdr**). ++ * ++ * Return ++ * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative ++ * error otherwise. ++ * ++ * int bpf_sysctl_get_name(struct bpf_sysctl *ctx, char *buf, size_t buf_len, u64 flags) ++ * Description ++ * Get name of sysctl in /proc/sys/ and copy it into provided by ++ * program buffer *buf* of size *buf_len*. ++ * ++ * The buffer is always NUL terminated, unless it's zero-sized. ++ * ++ * If *flags* is zero, full name (e.g. "net/ipv4/tcp_mem") is ++ * copied. Use **BPF_F_SYSCTL_BASE_NAME** flag to copy base name ++ * only (e.g. "tcp_mem"). ++ * Return ++ * Number of character copied (not including the trailing NUL). ++ * ++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain ++ * truncated name in this case). ++ * ++ * int bpf_sysctl_get_current_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) ++ * Description ++ * Get current value of sysctl as it is presented in /proc/sys ++ * (incl. newline, etc), and copy it as a string into provided ++ * by program buffer *buf* of size *buf_len*. ++ * ++ * The whole value is copied, no matter what file position user ++ * space issued e.g. sys_read at. ++ * ++ * The buffer is always NUL terminated, unless it's zero-sized. ++ * Return ++ * Number of character copied (not including the trailing NUL). ++ * ++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain ++ * truncated name in this case). ++ * ++ * **-EINVAL** if current value was unavailable, e.g. because ++ * sysctl is uninitialized and read returns -EIO for it. ++ * ++ * int bpf_sysctl_get_new_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) ++ * Description ++ * Get new value being written by user space to sysctl (before ++ * the actual write happens) and copy it as a string into ++ * provided by program buffer *buf* of size *buf_len*. ++ * ++ * User space may write new value at file position > 0. ++ * ++ * The buffer is always NUL terminated, unless it's zero-sized. ++ * Return ++ * Number of character copied (not including the trailing NUL). ++ * ++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain ++ * truncated name in this case). ++ * ++ * **-EINVAL** if sysctl is being read. ++ * ++ * int bpf_sysctl_set_new_value(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) ++ * Description ++ * Override new value being written by user space to sysctl with ++ * value provided by program in buffer *buf* of size *buf_len*. ++ * ++ * *buf* should contain a string in same form as provided by user ++ * space on sysctl write. ++ * ++ * User space may write new value at file position > 0. To override ++ * the whole sysctl value file position should be set to zero. ++ * Return ++ * 0 on success. ++ * ++ * **-E2BIG** if the *buf_len* is too big. ++ * ++ * **-EINVAL** if sysctl is being read. ++ * ++ * int bpf_strtol(const char *buf, size_t buf_len, u64 flags, long *res) ++ * Description ++ * Convert the initial part of the string from buffer *buf* of ++ * size *buf_len* to a long integer according to the given base ++ * and save the result in *res*. ++ * ++ * The string may begin with an arbitrary amount of white space ++ * (as determined by **isspace**\ (3)) followed by a single ++ * optional '**-**' sign. ++ * ++ * Five least significant bits of *flags* encode base, other bits ++ * are currently unused. ++ * ++ * Base must be either 8, 10, 16 or 0 to detect it automatically ++ * similar to user space **strtol**\ (3). ++ * Return ++ * Number of characters consumed on success. Must be positive but ++ * no more than *buf_len*. ++ * ++ * **-EINVAL** if no valid digits were found or unsupported base ++ * was provided. ++ * ++ * **-ERANGE** if resulting value was out of range. ++ * ++ * int bpf_strtoul(const char *buf, size_t buf_len, u64 flags, unsigned long *res) ++ * Description ++ * Convert the initial part of the string from buffer *buf* of ++ * size *buf_len* to an unsigned long integer according to the ++ * given base and save the result in *res*. ++ * ++ * The string may begin with an arbitrary amount of white space ++ * (as determined by **isspace**\ (3)). ++ * ++ * Five least significant bits of *flags* encode base, other bits ++ * are currently unused. ++ * ++ * Base must be either 8, 10, 16 or 0 to detect it automatically ++ * similar to user space **strtoul**\ (3). ++ * Return ++ * Number of characters consumed on success. Must be positive but ++ * no more than *buf_len*. ++ * ++ * **-EINVAL** if no valid digits were found or unsupported base ++ * was provided. ++ * ++ * **-ERANGE** if resulting value was out of range. ++ * ++ * void *bpf_sk_storage_get(struct bpf_map *map, struct bpf_sock *sk, void *value, u64 flags) ++ * Description ++ * Get a bpf-local-storage from a *sk*. ++ * ++ * Logically, it could be thought of getting the value from ++ * a *map* with *sk* as the **key**. From this ++ * perspective, the usage is not much different from ++ * **bpf_map_lookup_elem**\ (*map*, **&**\ *sk*) except this ++ * helper enforces the key must be a full socket and the map must ++ * be a **BPF_MAP_TYPE_SK_STORAGE** also. ++ * ++ * Underneath, the value is stored locally at *sk* instead of ++ * the *map*. The *map* is used as the bpf-local-storage ++ * "type". The bpf-local-storage "type" (i.e. the *map*) is ++ * searched against all bpf-local-storages residing at *sk*. ++ * ++ * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be ++ * used such that a new bpf-local-storage will be ++ * created if one does not exist. *value* can be used ++ * together with **BPF_SK_STORAGE_GET_F_CREATE** to specify ++ * the initial value of a bpf-local-storage. If *value* is ++ * **NULL**, the new bpf-local-storage will be zero initialized. ++ * Return ++ * A bpf-local-storage pointer is returned on success. ++ * ++ * **NULL** if not found or there was an error in adding ++ * a new bpf-local-storage. ++ * ++ * int bpf_sk_storage_delete(struct bpf_map *map, struct bpf_sock *sk) ++ * Description ++ * Delete a bpf-local-storage from a *sk*. ++ * Return ++ * 0 on success. ++ * ++ * **-ENOENT** if the bpf-local-storage cannot be found. ++ * ++ * int bpf_send_signal(u32 sig) ++ * Description ++ * Send signal *sig* to the current task. ++ * Return ++ * 0 on success or successfully queued. ++ * ++ * **-EBUSY** if work queue under nmi is full. ++ * ++ * **-EINVAL** if *sig* is invalid. ++ * ++ * **-EPERM** if no permission to send the *sig*. ++ * ++ * **-EAGAIN** if bpf program can try again. ++ * ++ * s64 bpf_tcp_gen_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) ++ * Description ++ * Try to issue a SYN cookie for the packet with corresponding ++ * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. ++ * ++ * *iph* points to the start of the IPv4 or IPv6 header, while ++ * *iph_len* contains **sizeof**\ (**struct iphdr**) or ++ * **sizeof**\ (**struct ip6hdr**). ++ * ++ * *th* points to the start of the TCP header, while *th_len* ++ * contains the length of the TCP header. ++ * ++ * Return ++ * On success, lower 32 bits hold the generated SYN cookie in ++ * followed by 16 bits which hold the MSS value for that cookie, ++ * and the top 16 bits are unused. ++ * ++ * On failure, the returned value is one of the following: ++ * ++ * **-EINVAL** SYN cookie cannot be issued due to error ++ * ++ * **-ENOENT** SYN cookie should not be issued (no SYN flood) ++ * ++ * **-EOPNOTSUPP** kernel configuration does not enable SYN cookies ++ * ++ * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 ++ */ ++#define __BPF_FUNC_MAPPER(FN) \ ++ FN(unspec), \ ++ FN(map_lookup_elem), \ ++ FN(map_update_elem), \ ++ FN(map_delete_elem), \ ++ FN(probe_read), \ ++ FN(ktime_get_ns), \ ++ FN(trace_printk), \ ++ FN(get_prandom_u32), \ ++ FN(get_smp_processor_id), \ ++ FN(skb_store_bytes), \ ++ FN(l3_csum_replace), \ ++ FN(l4_csum_replace), \ ++ FN(tail_call), \ ++ FN(clone_redirect), \ ++ FN(get_current_pid_tgid), \ ++ FN(get_current_uid_gid), \ ++ FN(get_current_comm), \ ++ FN(get_cgroup_classid), \ ++ FN(skb_vlan_push), \ ++ FN(skb_vlan_pop), \ ++ FN(skb_get_tunnel_key), \ ++ FN(skb_set_tunnel_key), \ ++ FN(perf_event_read), \ ++ FN(redirect), \ ++ FN(get_route_realm), \ ++ FN(perf_event_output), \ ++ FN(skb_load_bytes), \ ++ FN(get_stackid), \ ++ FN(csum_diff), \ ++ FN(skb_get_tunnel_opt), \ ++ FN(skb_set_tunnel_opt), \ ++ FN(skb_change_proto), \ ++ FN(skb_change_type), \ ++ FN(skb_under_cgroup), \ ++ FN(get_hash_recalc), \ ++ FN(get_current_task), \ ++ FN(probe_write_user), \ ++ FN(current_task_under_cgroup), \ ++ FN(skb_change_tail), \ ++ FN(skb_pull_data), \ ++ FN(csum_update), \ ++ FN(set_hash_invalid), \ ++ FN(get_numa_node_id), \ ++ FN(skb_change_head), \ ++ FN(xdp_adjust_head), \ ++ FN(probe_read_str), \ ++ FN(get_socket_cookie), \ ++ FN(get_socket_uid), \ ++ FN(set_hash), \ ++ FN(setsockopt), \ ++ FN(skb_adjust_room), \ ++ FN(redirect_map), \ ++ FN(sk_redirect_map), \ ++ FN(sock_map_update), \ ++ FN(xdp_adjust_meta), \ ++ FN(perf_event_read_value), \ ++ FN(perf_prog_read_value), \ ++ FN(getsockopt), \ ++ FN(override_return), \ ++ FN(sock_ops_cb_flags_set), \ ++ FN(msg_redirect_map), \ ++ FN(msg_apply_bytes), \ ++ FN(msg_cork_bytes), \ ++ FN(msg_pull_data), \ ++ FN(bind), \ ++ FN(xdp_adjust_tail), \ ++ FN(skb_get_xfrm_state), \ ++ FN(get_stack), \ ++ FN(skb_load_bytes_relative), \ ++ FN(fib_lookup), \ ++ FN(sock_hash_update), \ ++ FN(msg_redirect_hash), \ ++ FN(sk_redirect_hash), \ ++ FN(lwt_push_encap), \ ++ FN(lwt_seg6_store_bytes), \ ++ FN(lwt_seg6_adjust_srh), \ ++ FN(lwt_seg6_action), \ ++ FN(rc_repeat), \ ++ FN(rc_keydown), \ ++ FN(skb_cgroup_id), \ ++ FN(get_current_cgroup_id), \ ++ FN(get_local_storage), \ ++ FN(sk_select_reuseport), \ ++ FN(skb_ancestor_cgroup_id), \ ++ FN(sk_lookup_tcp), \ ++ FN(sk_lookup_udp), \ ++ FN(sk_release), \ ++ FN(map_push_elem), \ ++ FN(map_pop_elem), \ ++ FN(map_peek_elem), \ ++ FN(msg_push_data), \ ++ FN(msg_pop_data), \ ++ FN(rc_pointer_rel), \ ++ FN(spin_lock), \ ++ FN(spin_unlock), \ ++ FN(sk_fullsock), \ ++ FN(tcp_sock), \ ++ FN(skb_ecn_set_ce), \ ++ FN(get_listener_sock), \ ++ FN(skc_lookup_tcp), \ ++ FN(tcp_check_syncookie), \ ++ FN(sysctl_get_name), \ ++ FN(sysctl_get_current_value), \ ++ FN(sysctl_get_new_value), \ ++ FN(sysctl_set_new_value), \ ++ FN(strtol), \ ++ FN(strtoul), \ ++ FN(sk_storage_get), \ ++ FN(sk_storage_delete), \ ++ FN(send_signal), \ ++ FN(tcp_gen_syncookie), ++ + /* integer value in 'imm' field of BPF_CALL instruction selects which helper + * function eBPF program intends to call + */ ++#define __BPF_ENUM_FN(x) BPF_FUNC_ ## x + enum bpf_func_id { +- BPF_FUNC_unspec, +- BPF_FUNC_map_lookup_elem, /* void *map_lookup_elem(&map, &key) */ +- BPF_FUNC_map_update_elem, /* int map_update_elem(&map, &key, &value, flags) */ +- BPF_FUNC_map_delete_elem, /* int map_delete_elem(&map, &key) */ +- BPF_FUNC_probe_read, /* int bpf_probe_read(void *dst, int size, void *src) */ +- BPF_FUNC_ktime_get_ns, /* u64 bpf_ktime_get_ns(void) */ +- BPF_FUNC_trace_printk, /* int bpf_trace_printk(const char *fmt, int fmt_size, ...) */ +- BPF_FUNC_get_prandom_u32, /* u32 prandom_u32(void) */ +- BPF_FUNC_get_smp_processor_id, /* u32 raw_smp_processor_id(void) */ +- +- /** +- * skb_store_bytes(skb, offset, from, len, flags) - store bytes into packet +- * @skb: pointer to skb +- * @offset: offset within packet from skb->mac_header +- * @from: pointer where to copy bytes from +- * @len: number of bytes to store into packet +- * @flags: bit 0 - if true, recompute skb->csum +- * other bits - reserved +- * Return: 0 on success +- */ +- BPF_FUNC_skb_store_bytes, +- +- /** +- * l3_csum_replace(skb, offset, from, to, flags) - recompute IP checksum +- * @skb: pointer to skb +- * @offset: offset within packet where IP checksum is located +- * @from: old value of header field +- * @to: new value of header field +- * @flags: bits 0-3 - size of header field +- * other bits - reserved +- * Return: 0 on success +- */ +- BPF_FUNC_l3_csum_replace, +- +- /** +- * l4_csum_replace(skb, offset, from, to, flags) - recompute TCP/UDP checksum +- * @skb: pointer to skb +- * @offset: offset within packet where TCP/UDP checksum is located +- * @from: old value of header field +- * @to: new value of header field +- * @flags: bits 0-3 - size of header field +- * bit 4 - is pseudo header +- * other bits - reserved +- * Return: 0 on success +- */ +- BPF_FUNC_l4_csum_replace, ++ __BPF_FUNC_MAPPER(__BPF_ENUM_FN) ++ __BPF_FUNC_MAX_ID, ++}; ++#undef __BPF_ENUM_FN + +- /** +- * bpf_tail_call(ctx, prog_array_map, index) - jump into another BPF program +- * @ctx: context pointer passed to next program +- * @prog_array_map: pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY +- * @index: index inside array that selects specific program to run +- * Return: 0 on success +- */ +- BPF_FUNC_tail_call, ++/* All flags used by eBPF helper functions, placed here. */ + +- /** +- * bpf_clone_redirect(skb, ifindex, flags) - redirect to another netdev +- * @skb: pointer to skb +- * @ifindex: ifindex of the net device +- * @flags: bit 0 - if set, redirect to ingress instead of egress +- * other bits - reserved +- * Return: 0 on success +- */ +- BPF_FUNC_clone_redirect, ++/* BPF_FUNC_skb_store_bytes flags. */ ++#define BPF_F_RECOMPUTE_CSUM (1ULL << 0) ++#define BPF_F_INVALIDATE_HASH (1ULL << 1) + +- /** +- * u64 bpf_get_current_pid_tgid(void) +- * Return: current->tgid << 32 | current->pid +- */ +- BPF_FUNC_get_current_pid_tgid, ++/* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags. ++ * First 4 bits are for passing the header field size. ++ */ ++#define BPF_F_HDR_FIELD_MASK 0xfULL + +- /** +- * u64 bpf_get_current_uid_gid(void) +- * Return: current_gid << 32 | current_uid +- */ +- BPF_FUNC_get_current_uid_gid, ++/* BPF_FUNC_l4_csum_replace flags. */ ++#define BPF_F_PSEUDO_HDR (1ULL << 4) ++#define BPF_F_MARK_MANGLED_0 (1ULL << 5) ++#define BPF_F_MARK_ENFORCE (1ULL << 6) ++ ++/* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ ++#define BPF_F_INGRESS (1ULL << 0) ++ ++/* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */ ++#define BPF_F_TUNINFO_IPV6 (1ULL << 0) ++ ++/* flags for both BPF_FUNC_get_stackid and BPF_FUNC_get_stack. */ ++#define BPF_F_SKIP_FIELD_MASK 0xffULL ++#define BPF_F_USER_STACK (1ULL << 8) ++/* flags used by BPF_FUNC_get_stackid only. */ ++#define BPF_F_FAST_STACK_CMP (1ULL << 9) ++#define BPF_F_REUSE_STACKID (1ULL << 10) ++/* flags used by BPF_FUNC_get_stack only. */ ++#define BPF_F_USER_BUILD_ID (1ULL << 11) ++ ++/* BPF_FUNC_skb_set_tunnel_key flags. */ ++#define BPF_F_ZERO_CSUM_TX (1ULL << 1) ++#define BPF_F_DONT_FRAGMENT (1ULL << 2) ++#define BPF_F_SEQ_NUMBER (1ULL << 3) + +- /** +- * bpf_get_current_comm(char *buf, int size_of_buf) +- * stores current->comm into buf +- * Return: 0 on success +- */ +- BPF_FUNC_get_current_comm, +- +- /** +- * bpf_get_cgroup_classid(skb) - retrieve a proc's classid +- * @skb: pointer to skb +- * Return: classid if != 0 +- */ +- BPF_FUNC_get_cgroup_classid, +- BPF_FUNC_skb_vlan_push, /* bpf_skb_vlan_push(skb, vlan_proto, vlan_tci) */ +- BPF_FUNC_skb_vlan_pop, /* bpf_skb_vlan_pop(skb) */ +- +- /** +- * bpf_skb_[gs]et_tunnel_key(skb, key, size, flags) +- * retrieve or populate tunnel metadata +- * @skb: pointer to skb +- * @key: pointer to 'struct bpf_tunnel_key' +- * @size: size of 'struct bpf_tunnel_key' +- * @flags: room for future extensions +- * Retrun: 0 on success +- */ +- BPF_FUNC_skb_get_tunnel_key, +- BPF_FUNC_skb_set_tunnel_key, +- BPF_FUNC_perf_event_read, /* u64 bpf_perf_event_read(&map, index) */ +- /** +- * bpf_redirect(ifindex, flags) - redirect to another netdev +- * @ifindex: ifindex of the net device +- * @flags: bit 0 - if set, redirect to ingress instead of egress +- * other bits - reserved +- * Return: TC_ACT_REDIRECT +- */ +- BPF_FUNC_redirect, ++/* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and ++ * BPF_FUNC_perf_event_read_value flags. ++ */ ++#define BPF_F_INDEX_MASK 0xffffffffULL ++#define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK ++/* BPF_FUNC_perf_event_output for sk_buff input context. */ ++#define BPF_F_CTXLEN_MASK (0xfffffULL << 32) ++ ++/* Current network namespace */ ++#define BPF_F_CURRENT_NETNS (-1L) ++ ++/* BPF_FUNC_skb_adjust_room flags. */ ++#define BPF_F_ADJ_ROOM_FIXED_GSO (1ULL << 0) ++ ++#define BPF_ADJ_ROOM_ENCAP_L2_MASK 0xff ++#define BPF_ADJ_ROOM_ENCAP_L2_SHIFT 56 ++ ++#define BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 (1ULL << 1) ++#define BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 (1ULL << 2) ++#define BPF_F_ADJ_ROOM_ENCAP_L4_GRE (1ULL << 3) ++#define BPF_F_ADJ_ROOM_ENCAP_L4_UDP (1ULL << 4) ++#define BPF_F_ADJ_ROOM_ENCAP_L2(len) (((__u64)len & \ ++ BPF_ADJ_ROOM_ENCAP_L2_MASK) \ ++ << BPF_ADJ_ROOM_ENCAP_L2_SHIFT) ++ ++/* BPF_FUNC_sysctl_get_name flags. */ ++#define BPF_F_SYSCTL_BASE_NAME (1ULL << 0) ++ ++/* BPF_FUNC_sk_storage_get flags */ ++#define BPF_SK_STORAGE_GET_F_CREATE (1ULL << 0) ++ ++/* Mode for BPF_FUNC_skb_adjust_room helper. */ ++enum bpf_adj_room_mode { ++ BPF_ADJ_ROOM_NET, ++ BPF_ADJ_ROOM_MAC, ++}; + +- /** +- * bpf_get_route_realm(skb) - retrieve a dst's tclassid +- * @skb: pointer to skb +- * Return: realm if != 0 +- */ +- BPF_FUNC_get_route_realm, ++/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */ ++enum bpf_hdr_start_off { ++ BPF_HDR_START_MAC, ++ BPF_HDR_START_NET, ++}; + +- /** +- * bpf_perf_event_output(ctx, map, index, data, size) - output perf raw sample +- * @ctx: struct pt_regs* +- * @map: pointer to perf_event_array map +- * @index: index of event in the map +- * @data: data on stack to be output as raw data +- * @size: size of data +- * Return: 0 on success +- */ +- BPF_FUNC_perf_event_output, +- __BPF_FUNC_MAX_ID, ++/* Encapsulation type for BPF_FUNC_lwt_push_encap helper. */ ++enum bpf_lwt_encap_mode { ++ BPF_LWT_ENCAP_SEG6, ++ BPF_LWT_ENCAP_SEG6_INLINE, ++ BPF_LWT_ENCAP_IP, + }; + ++#define __bpf_md_ptr(type, name) \ ++union { \ ++ type name; \ ++ __u64 :64; \ ++} __attribute__((aligned(8))) ++ + /* user accessible mirror of in-kernel sk_buff. + * new fields can only be added to the end of this structure + */ +@@ -291,11 +2985,632 @@ struct __sk_buff { + __u32 cb[5]; + __u32 hash; + __u32 tc_classid; ++ __u32 data; ++ __u32 data_end; ++ __u32 napi_id; ++ ++ /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ ++ __u32 family; ++ __u32 remote_ip4; /* Stored in network byte order */ ++ __u32 local_ip4; /* Stored in network byte order */ ++ __u32 remote_ip6[4]; /* Stored in network byte order */ ++ __u32 local_ip6[4]; /* Stored in network byte order */ ++ __u32 remote_port; /* Stored in network byte order */ ++ __u32 local_port; /* stored in host byte order */ ++ /* ... here. */ ++ ++ __u32 data_meta; ++ __bpf_md_ptr(struct bpf_flow_keys *, flow_keys); ++ __u64 tstamp; ++ __u32 wire_len; ++ __u32 gso_segs; ++ __bpf_md_ptr(struct bpf_sock *, sk); + }; + + struct bpf_tunnel_key { + __u32 tunnel_id; +- __u32 remote_ipv4; ++ union { ++ __u32 remote_ipv4; ++ __u32 remote_ipv6[4]; ++ }; ++ __u8 tunnel_tos; ++ __u8 tunnel_ttl; ++ __u16 tunnel_ext; /* Padding, future use. */ ++ __u32 tunnel_label; ++}; ++ ++/* user accessible mirror of in-kernel xfrm_state. ++ * new fields can only be added to the end of this structure ++ */ ++struct bpf_xfrm_state { ++ __u32 reqid; ++ __u32 spi; /* Stored in network byte order */ ++ __u16 family; ++ __u16 ext; /* Padding, future use. */ ++ union { ++ __u32 remote_ipv4; /* Stored in network byte order */ ++ __u32 remote_ipv6[4]; /* Stored in network byte order */ ++ }; ++}; ++ ++/* Generic BPF return codes which all BPF program types may support. ++ * The values are binary compatible with their TC_ACT_* counter-part to ++ * provide backwards compatibility with existing SCHED_CLS and SCHED_ACT ++ * programs. ++ * ++ * XDP is handled seprately, see XDP_*. ++ */ ++enum bpf_ret_code { ++ BPF_OK = 0, ++ /* 1 reserved */ ++ BPF_DROP = 2, ++ /* 3-6 reserved */ ++ BPF_REDIRECT = 7, ++ /* >127 are reserved for prog type specific return codes. ++ * ++ * BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and ++ * BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been ++ * changed and should be routed based on its new L3 header. ++ * (This is an L3 redirect, as opposed to L2 redirect ++ * represented by BPF_REDIRECT above). ++ */ ++ BPF_LWT_REROUTE = 128, ++}; ++ ++struct bpf_sock { ++ __u32 bound_dev_if; ++ __u32 family; ++ __u32 type; ++ __u32 protocol; ++ __u32 mark; ++ __u32 priority; ++ /* IP address also allows 1 and 2 bytes access */ ++ __u32 src_ip4; ++ __u32 src_ip6[4]; ++ __u32 src_port; /* host byte order */ ++ __u32 dst_port; /* network byte order */ ++ __u32 dst_ip4; ++ __u32 dst_ip6[4]; ++ __u32 state; ++}; ++ ++struct bpf_tcp_sock { ++ __u32 snd_cwnd; /* Sending congestion window */ ++ __u32 srtt_us; /* smoothed round trip time << 3 in usecs */ ++ __u32 rtt_min; ++ __u32 snd_ssthresh; /* Slow start size threshold */ ++ __u32 rcv_nxt; /* What we want to receive next */ ++ __u32 snd_nxt; /* Next sequence we send */ ++ __u32 snd_una; /* First byte we want an ack for */ ++ __u32 mss_cache; /* Cached effective mss, not including SACKS */ ++ __u32 ecn_flags; /* ECN status bits. */ ++ __u32 rate_delivered; /* saved rate sample: packets delivered */ ++ __u32 rate_interval_us; /* saved rate sample: time elapsed */ ++ __u32 packets_out; /* Packets which are "in flight" */ ++ __u32 retrans_out; /* Retransmitted packets out */ ++ __u32 total_retrans; /* Total retransmits for entire connection */ ++ __u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn ++ * total number of segments in. ++ */ ++ __u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn ++ * total number of data segments in. ++ */ ++ __u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut ++ * The total number of segments sent. ++ */ ++ __u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut ++ * total number of data segments sent. ++ */ ++ __u32 lost_out; /* Lost packets */ ++ __u32 sacked_out; /* SACK'd packets */ ++ __u64 bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived ++ * sum(delta(rcv_nxt)), or how many bytes ++ * were acked. ++ */ ++ __u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked ++ * sum(delta(snd_una)), or how many bytes ++ * were acked. ++ */ ++ __u32 dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups ++ * total number of DSACK blocks received ++ */ ++ __u32 delivered; /* Total data packets delivered incl. rexmits */ ++ __u32 delivered_ce; /* Like the above but only ECE marked packets */ ++ __u32 icsk_retransmits; /* Number of unrecovered [RTO] timeouts */ ++}; ++ ++struct bpf_sock_tuple { ++ union { ++ struct { ++ __be32 saddr; ++ __be32 daddr; ++ __be16 sport; ++ __be16 dport; ++ } ipv4; ++ struct { ++ __be32 saddr[4]; ++ __be32 daddr[4]; ++ __be16 sport; ++ __be16 dport; ++ } ipv6; ++ }; ++}; ++ ++struct bpf_xdp_sock { ++ __u32 queue_id; ++}; ++ ++#define XDP_PACKET_HEADROOM 256 ++ ++/* User return codes for XDP prog type. ++ * A valid XDP program must return one of these defined values. All other ++ * return codes are reserved for future use. Unknown return codes will ++ * result in packet drops and a warning via bpf_warn_invalid_xdp_action(). ++ */ ++enum xdp_action { ++ XDP_ABORTED = 0, ++ XDP_DROP, ++ XDP_PASS, ++ XDP_TX, ++ XDP_REDIRECT, ++}; ++ ++/* user accessible metadata for XDP packet hook ++ * new fields must be added to the end of this structure ++ */ ++struct xdp_md { ++ __u32 data; ++ __u32 data_end; ++ __u32 data_meta; ++ /* Below access go through struct xdp_rxq_info */ ++ __u32 ingress_ifindex; /* rxq->dev->ifindex */ ++ __u32 rx_queue_index; /* rxq->queue_index */ ++}; ++ ++enum sk_action { ++ SK_DROP = 0, ++ SK_PASS, ++}; ++ ++/* user accessible metadata for SK_MSG packet hook, new fields must ++ * be added to the end of this structure ++ */ ++struct sk_msg_md { ++ __bpf_md_ptr(void *, data); ++ __bpf_md_ptr(void *, data_end); ++ ++ __u32 family; ++ __u32 remote_ip4; /* Stored in network byte order */ ++ __u32 local_ip4; /* Stored in network byte order */ ++ __u32 remote_ip6[4]; /* Stored in network byte order */ ++ __u32 local_ip6[4]; /* Stored in network byte order */ ++ __u32 remote_port; /* Stored in network byte order */ ++ __u32 local_port; /* stored in host byte order */ ++ __u32 size; /* Total size of sk_msg */ ++}; ++ ++struct sk_reuseport_md { ++ /* ++ * Start of directly accessible data. It begins from ++ * the tcp/udp header. ++ */ ++ __bpf_md_ptr(void *, data); ++ /* End of directly accessible data */ ++ __bpf_md_ptr(void *, data_end); ++ /* ++ * Total length of packet (starting from the tcp/udp header). ++ * Note that the directly accessible bytes (data_end - data) ++ * could be less than this "len". Those bytes could be ++ * indirectly read by a helper "bpf_skb_load_bytes()". ++ */ ++ __u32 len; ++ /* ++ * Eth protocol in the mac header (network byte order). e.g. ++ * ETH_P_IP(0x0800) and ETH_P_IPV6(0x86DD) ++ */ ++ __u32 eth_protocol; ++ __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ ++ __u32 bind_inany; /* Is sock bound to an INANY address? */ ++ __u32 hash; /* A hash of the packet 4 tuples */ ++}; ++ ++#define BPF_TAG_SIZE 8 ++ ++struct bpf_prog_info { ++ __u32 type; ++ __u32 id; ++ __u8 tag[BPF_TAG_SIZE]; ++ __u32 jited_prog_len; ++ __u32 xlated_prog_len; ++ __aligned_u64 jited_prog_insns; ++ __aligned_u64 xlated_prog_insns; ++ __u64 load_time; /* ns since boottime */ ++ __u32 created_by_uid; ++ __u32 nr_map_ids; ++ __aligned_u64 map_ids; ++ char name[BPF_OBJ_NAME_LEN]; ++ __u32 ifindex; ++ __u32 gpl_compatible:1; ++ __u32 :31; /* alignment pad */ ++ __u64 netns_dev; ++ __u64 netns_ino; ++ __u32 nr_jited_ksyms; ++ __u32 nr_jited_func_lens; ++ __aligned_u64 jited_ksyms; ++ __aligned_u64 jited_func_lens; ++ __u32 btf_id; ++ __u32 func_info_rec_size; ++ __aligned_u64 func_info; ++ __u32 nr_func_info; ++ __u32 nr_line_info; ++ __aligned_u64 line_info; ++ __aligned_u64 jited_line_info; ++ __u32 nr_jited_line_info; ++ __u32 line_info_rec_size; ++ __u32 jited_line_info_rec_size; ++ __u32 nr_prog_tags; ++ __aligned_u64 prog_tags; ++ __u64 run_time_ns; ++ __u64 run_cnt; ++} __attribute__((aligned(8))); ++ ++struct bpf_map_info { ++ __u32 type; ++ __u32 id; ++ __u32 key_size; ++ __u32 value_size; ++ __u32 max_entries; ++ __u32 map_flags; ++ char name[BPF_OBJ_NAME_LEN]; ++ __u32 ifindex; ++ __u32 :32; ++ __u64 netns_dev; ++ __u64 netns_ino; ++ __u32 btf_id; ++ __u32 btf_key_type_id; ++ __u32 btf_value_type_id; ++} __attribute__((aligned(8))); ++ ++struct bpf_btf_info { ++ __aligned_u64 btf; ++ __u32 btf_size; ++ __u32 id; ++} __attribute__((aligned(8))); ++ ++/* User bpf_sock_addr struct to access socket fields and sockaddr struct passed ++ * by user and intended to be used by socket (e.g. to bind to, depends on ++ * attach attach type). ++ */ ++struct bpf_sock_addr { ++ __u32 user_family; /* Allows 4-byte read, but no write. */ ++ __u32 user_ip4; /* Allows 1,2,4-byte read and 4-byte write. ++ * Stored in network byte order. ++ */ ++ __u32 user_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. ++ * Stored in network byte order. ++ */ ++ __u32 user_port; /* Allows 4-byte read and write. ++ * Stored in network byte order ++ */ ++ __u32 family; /* Allows 4-byte read, but no write */ ++ __u32 type; /* Allows 4-byte read, but no write */ ++ __u32 protocol; /* Allows 4-byte read, but no write */ ++ __u32 msg_src_ip4; /* Allows 1,2,4-byte read and 4-byte write. ++ * Stored in network byte order. ++ */ ++ __u32 msg_src_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. ++ * Stored in network byte order. ++ */ ++ __bpf_md_ptr(struct bpf_sock *, sk); ++}; ++ ++/* User bpf_sock_ops struct to access socket values and specify request ops ++ * and their replies. ++ * Some of this fields are in network (bigendian) byte order and may need ++ * to be converted before use (bpf_ntohl() defined in samples/bpf/bpf_endian.h). ++ * New fields can only be added at the end of this structure ++ */ ++struct bpf_sock_ops { ++ __u32 op; ++ union { ++ __u32 args[4]; /* Optionally passed to bpf program */ ++ __u32 reply; /* Returned by bpf program */ ++ __u32 replylong[4]; /* Optionally returned by bpf prog */ ++ }; ++ __u32 family; ++ __u32 remote_ip4; /* Stored in network byte order */ ++ __u32 local_ip4; /* Stored in network byte order */ ++ __u32 remote_ip6[4]; /* Stored in network byte order */ ++ __u32 local_ip6[4]; /* Stored in network byte order */ ++ __u32 remote_port; /* Stored in network byte order */ ++ __u32 local_port; /* stored in host byte order */ ++ __u32 is_fullsock; /* Some TCP fields are only valid if ++ * there is a full socket. If not, the ++ * fields read as zero. ++ */ ++ __u32 snd_cwnd; ++ __u32 srtt_us; /* Averaged RTT << 3 in usecs */ ++ __u32 bpf_sock_ops_cb_flags; /* flags defined in uapi/linux/tcp.h */ ++ __u32 state; ++ __u32 rtt_min; ++ __u32 snd_ssthresh; ++ __u32 rcv_nxt; ++ __u32 snd_nxt; ++ __u32 snd_una; ++ __u32 mss_cache; ++ __u32 ecn_flags; ++ __u32 rate_delivered; ++ __u32 rate_interval_us; ++ __u32 packets_out; ++ __u32 retrans_out; ++ __u32 total_retrans; ++ __u32 segs_in; ++ __u32 data_segs_in; ++ __u32 segs_out; ++ __u32 data_segs_out; ++ __u32 lost_out; ++ __u32 sacked_out; ++ __u32 sk_txhash; ++ __u64 bytes_received; ++ __u64 bytes_acked; ++ __bpf_md_ptr(struct bpf_sock *, sk); ++}; ++ ++/* Definitions for bpf_sock_ops_cb_flags */ ++#define BPF_SOCK_OPS_RTO_CB_FLAG (1<<0) ++#define BPF_SOCK_OPS_RETRANS_CB_FLAG (1<<1) ++#define BPF_SOCK_OPS_STATE_CB_FLAG (1<<2) ++#define BPF_SOCK_OPS_RTT_CB_FLAG (1<<3) ++#define BPF_SOCK_OPS_ALL_CB_FLAGS 0xF /* Mask of all currently ++ * supported cb flags ++ */ ++ ++/* List of known BPF sock_ops operators. ++ * New entries can only be added at the end ++ */ ++enum { ++ BPF_SOCK_OPS_VOID, ++ BPF_SOCK_OPS_TIMEOUT_INIT, /* Should return SYN-RTO value to use or ++ * -1 if default value should be used ++ */ ++ BPF_SOCK_OPS_RWND_INIT, /* Should return initial advertized ++ * window (in packets) or -1 if default ++ * value should be used ++ */ ++ BPF_SOCK_OPS_TCP_CONNECT_CB, /* Calls BPF program right before an ++ * active connection is initialized ++ */ ++ BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, /* Calls BPF program when an ++ * active connection is ++ * established ++ */ ++ BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB, /* Calls BPF program when a ++ * passive connection is ++ * established ++ */ ++ BPF_SOCK_OPS_NEEDS_ECN, /* If connection's congestion control ++ * needs ECN ++ */ ++ BPF_SOCK_OPS_BASE_RTT, /* Get base RTT. The correct value is ++ * based on the path and may be ++ * dependent on the congestion control ++ * algorithm. In general it indicates ++ * a congestion threshold. RTTs above ++ * this indicate congestion ++ */ ++ BPF_SOCK_OPS_RTO_CB, /* Called when an RTO has triggered. ++ * Arg1: value of icsk_retransmits ++ * Arg2: value of icsk_rto ++ * Arg3: whether RTO has expired ++ */ ++ BPF_SOCK_OPS_RETRANS_CB, /* Called when skb is retransmitted. ++ * Arg1: sequence number of 1st byte ++ * Arg2: # segments ++ * Arg3: return value of ++ * tcp_transmit_skb (0 => success) ++ */ ++ BPF_SOCK_OPS_STATE_CB, /* Called when TCP changes state. ++ * Arg1: old_state ++ * Arg2: new_state ++ */ ++ BPF_SOCK_OPS_TCP_LISTEN_CB, /* Called on listen(2), right after ++ * socket transition to LISTEN state. ++ */ ++ BPF_SOCK_OPS_RTT_CB, /* Called on every RTT. ++ */ ++}; ++ ++/* List of TCP states. There is a build check in net/ipv4/tcp.c to detect ++ * changes between the TCP and BPF versions. Ideally this should never happen. ++ * If it does, we need to add code to convert them before calling ++ * the BPF sock_ops function. ++ */ ++enum { ++ BPF_TCP_ESTABLISHED = 1, ++ BPF_TCP_SYN_SENT, ++ BPF_TCP_SYN_RECV, ++ BPF_TCP_FIN_WAIT1, ++ BPF_TCP_FIN_WAIT2, ++ BPF_TCP_TIME_WAIT, ++ BPF_TCP_CLOSE, ++ BPF_TCP_CLOSE_WAIT, ++ BPF_TCP_LAST_ACK, ++ BPF_TCP_LISTEN, ++ BPF_TCP_CLOSING, /* Now a valid state */ ++ BPF_TCP_NEW_SYN_RECV, ++ ++ BPF_TCP_MAX_STATES /* Leave at the end! */ ++}; ++ ++#define TCP_BPF_IW 1001 /* Set TCP initial congestion window */ ++#define TCP_BPF_SNDCWND_CLAMP 1002 /* Set sndcwnd_clamp */ ++ ++struct bpf_perf_event_value { ++ __u64 counter; ++ __u64 enabled; ++ __u64 running; ++}; ++ ++#define BPF_DEVCG_ACC_MKNOD (1ULL << 0) ++#define BPF_DEVCG_ACC_READ (1ULL << 1) ++#define BPF_DEVCG_ACC_WRITE (1ULL << 2) ++ ++#define BPF_DEVCG_DEV_BLOCK (1ULL << 0) ++#define BPF_DEVCG_DEV_CHAR (1ULL << 1) ++ ++struct bpf_cgroup_dev_ctx { ++ /* access_type encoded as (BPF_DEVCG_ACC_* << 16) | BPF_DEVCG_DEV_* */ ++ __u32 access_type; ++ __u32 major; ++ __u32 minor; ++}; ++ ++struct bpf_raw_tracepoint_args { ++ __u64 args[0]; ++}; ++ ++/* DIRECT: Skip the FIB rules and go to FIB table associated with device ++ * OUTPUT: Do lookup from egress perspective; default is ingress ++ */ ++#define BPF_FIB_LOOKUP_DIRECT (1U << 0) ++#define BPF_FIB_LOOKUP_OUTPUT (1U << 1) ++ ++enum { ++ BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */ ++ BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */ ++ BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */ ++ BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */ ++ BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */ ++ BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */ ++ BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ ++ BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ ++ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ ++}; ++ ++struct bpf_fib_lookup { ++ /* input: network family for lookup (AF_INET, AF_INET6) ++ * output: network family of egress nexthop ++ */ ++ __u8 family; ++ ++ /* set if lookup is to consider L4 data - e.g., FIB rules */ ++ __u8 l4_protocol; ++ __be16 sport; ++ __be16 dport; ++ ++ /* total length of packet from network header - used for MTU check */ ++ __u16 tot_len; ++ ++ /* input: L3 device index for lookup ++ * output: device index from FIB lookup ++ */ ++ __u32 ifindex; ++ ++ union { ++ /* inputs to lookup */ ++ __u8 tos; /* AF_INET */ ++ __be32 flowinfo; /* AF_INET6, flow_label + priority */ ++ ++ /* output: metric of fib result (IPv4/IPv6 only) */ ++ __u32 rt_metric; ++ }; ++ ++ union { ++ __be32 ipv4_src; ++ __u32 ipv6_src[4]; /* in6_addr; network order */ ++ }; ++ ++ /* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in ++ * network header. output: bpf_fib_lookup sets to gateway address ++ * if FIB lookup returns gateway route ++ */ ++ union { ++ __be32 ipv4_dst; ++ __u32 ipv6_dst[4]; /* in6_addr; network order */ ++ }; ++ ++ /* output */ ++ __be16 h_vlan_proto; ++ __be16 h_vlan_TCI; ++ __u8 smac[6]; /* ETH_ALEN */ ++ __u8 dmac[6]; /* ETH_ALEN */ ++}; ++ ++enum bpf_task_fd_type { ++ BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */ ++ BPF_FD_TYPE_TRACEPOINT, /* tp name */ ++ BPF_FD_TYPE_KPROBE, /* (symbol + offset) or addr */ ++ BPF_FD_TYPE_KRETPROBE, /* (symbol + offset) or addr */ ++ BPF_FD_TYPE_UPROBE, /* filename + offset */ ++ BPF_FD_TYPE_URETPROBE, /* filename + offset */ ++}; ++ ++#define BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG (1U << 0) ++#define BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL (1U << 1) ++#define BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP (1U << 2) ++ ++struct bpf_flow_keys { ++ __u16 nhoff; ++ __u16 thoff; ++ __u16 addr_proto; /* ETH_P_* of valid addrs */ ++ __u8 is_frag; ++ __u8 is_first_frag; ++ __u8 is_encap; ++ __u8 ip_proto; ++ __be16 n_proto; ++ __be16 sport; ++ __be16 dport; ++ union { ++ struct { ++ __be32 ipv4_src; ++ __be32 ipv4_dst; ++ }; ++ struct { ++ __u32 ipv6_src[4]; /* in6_addr; network order */ ++ __u32 ipv6_dst[4]; /* in6_addr; network order */ ++ }; ++ }; ++ __u32 flags; ++ __be32 flow_label; ++}; ++ ++struct bpf_func_info { ++ __u32 insn_off; ++ __u32 type_id; ++}; ++ ++#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10) ++#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff) ++ ++struct bpf_line_info { ++ __u32 insn_off; ++ __u32 file_name_off; ++ __u32 line_off; ++ __u32 line_col; ++}; ++ ++struct bpf_spin_lock { ++ __u32 val; ++}; ++ ++struct bpf_sysctl { ++ __u32 write; /* Sysctl is being read (= 0) or written (= 1). ++ * Allows 1,2,4-byte read, but no write. ++ */ ++ __u32 file_pos; /* Sysctl file position to read from, write to. ++ * Allows 1,2,4-byte read an 4-byte write. ++ */ ++}; ++ ++struct bpf_sockopt { ++ __bpf_md_ptr(struct bpf_sock *, sk); ++ __bpf_md_ptr(void *, optval); ++ __bpf_md_ptr(void *, optval_end); ++ ++ __s32 level; ++ __s32 optname; ++ __s32 optlen; ++ __s32 retval; + }; + + #endif /* _UAPI__LINUX_BPF_H__ */ +--- /dev/null ++++ b/include/uapi/linux/bpfilter.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++#ifndef _UAPI_LINUX_BPFILTER_H ++#define _UAPI_LINUX_BPFILTER_H ++ ++#include ++ ++enum { ++ BPFILTER_IPT_SO_SET_REPLACE = 64, ++ BPFILTER_IPT_SO_SET_ADD_COUNTERS = 65, ++ BPFILTER_IPT_SET_MAX, ++}; ++ ++enum { ++ BPFILTER_IPT_SO_GET_INFO = 64, ++ BPFILTER_IPT_SO_GET_ENTRIES = 65, ++ BPFILTER_IPT_SO_GET_REVISION_MATCH = 66, ++ BPFILTER_IPT_SO_GET_REVISION_TARGET = 67, ++ BPFILTER_IPT_GET_MAX, ++}; ++ ++#endif /* _UAPI_LINUX_BPFILTER_H */ +--- /dev/null ++++ b/include/uapi/linux/bpf_perf_event.h +@@ -0,0 +1,19 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* Copyright (c) 2016 Facebook ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of version 2 of the GNU General Public ++ * License as published by the Free Software Foundation. ++ */ ++#ifndef _UAPI__LINUX_BPF_PERF_EVENT_H__ ++#define _UAPI__LINUX_BPF_PERF_EVENT_H__ ++ ++#include ++ ++struct bpf_perf_event_data { ++ bpf_user_pt_regs_t regs; ++ __u64 sample_period; ++ __u64 addr; ++}; ++ ++#endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */ +--- /dev/null ++++ b/include/uapi/linux/btf.h +@@ -0,0 +1,165 @@ ++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ ++/* Copyright (c) 2018 Facebook */ ++#ifndef _UAPI__LINUX_BTF_H__ ++#define _UAPI__LINUX_BTF_H__ ++ ++#include ++ ++#define BTF_MAGIC 0xeB9F ++#define BTF_VERSION 1 ++ ++struct btf_header { ++ __u16 magic; ++ __u8 version; ++ __u8 flags; ++ __u32 hdr_len; ++ ++ /* All offsets are in bytes relative to the end of this header */ ++ __u32 type_off; /* offset of type section */ ++ __u32 type_len; /* length of type section */ ++ __u32 str_off; /* offset of string section */ ++ __u32 str_len; /* length of string section */ ++}; ++ ++/* Max # of type identifier */ ++#define BTF_MAX_TYPE 0x000fffff ++/* Max offset into the string section */ ++#define BTF_MAX_NAME_OFFSET 0x00ffffff ++/* Max # of struct/union/enum members or func args */ ++#define BTF_MAX_VLEN 0xffff ++ ++struct btf_type { ++ __u32 name_off; ++ /* "info" bits arrangement ++ * bits 0-15: vlen (e.g. # of struct's members) ++ * bits 16-23: unused ++ * bits 24-27: kind (e.g. int, ptr, array...etc) ++ * bits 28-30: unused ++ * bit 31: kind_flag, currently used by ++ * struct, union and fwd ++ */ ++ __u32 info; ++ /* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC. ++ * "size" tells the size of the type it is describing. ++ * ++ * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, ++ * FUNC, FUNC_PROTO and VAR. ++ * "type" is a type_id referring to another type. ++ */ ++ union { ++ __u32 size; ++ __u32 type; ++ }; ++}; ++ ++#define BTF_INFO_KIND(info) (((info) >> 24) & 0x0f) ++#define BTF_INFO_VLEN(info) ((info) & 0xffff) ++#define BTF_INFO_KFLAG(info) ((info) >> 31) ++ ++#define BTF_KIND_UNKN 0 /* Unknown */ ++#define BTF_KIND_INT 1 /* Integer */ ++#define BTF_KIND_PTR 2 /* Pointer */ ++#define BTF_KIND_ARRAY 3 /* Array */ ++#define BTF_KIND_STRUCT 4 /* Struct */ ++#define BTF_KIND_UNION 5 /* Union */ ++#define BTF_KIND_ENUM 6 /* Enumeration */ ++#define BTF_KIND_FWD 7 /* Forward */ ++#define BTF_KIND_TYPEDEF 8 /* Typedef */ ++#define BTF_KIND_VOLATILE 9 /* Volatile */ ++#define BTF_KIND_CONST 10 /* Const */ ++#define BTF_KIND_RESTRICT 11 /* Restrict */ ++#define BTF_KIND_FUNC 12 /* Function */ ++#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ ++#define BTF_KIND_VAR 14 /* Variable */ ++#define BTF_KIND_DATASEC 15 /* Section */ ++#define BTF_KIND_MAX BTF_KIND_DATASEC ++#define NR_BTF_KINDS (BTF_KIND_MAX + 1) ++ ++/* For some specific BTF_KIND, "struct btf_type" is immediately ++ * followed by extra data. ++ */ ++ ++/* BTF_KIND_INT is followed by a u32 and the following ++ * is the 32 bits arrangement: ++ */ ++#define BTF_INT_ENCODING(VAL) (((VAL) & 0x0f000000) >> 24) ++#define BTF_INT_OFFSET(VAL) (((VAL) & 0x00ff0000) >> 16) ++#define BTF_INT_BITS(VAL) ((VAL) & 0x000000ff) ++ ++/* Attributes stored in the BTF_INT_ENCODING */ ++#define BTF_INT_SIGNED (1 << 0) ++#define BTF_INT_CHAR (1 << 1) ++#define BTF_INT_BOOL (1 << 2) ++ ++/* BTF_KIND_ENUM is followed by multiple "struct btf_enum". ++ * The exact number of btf_enum is stored in the vlen (of the ++ * info in "struct btf_type"). ++ */ ++struct btf_enum { ++ __u32 name_off; ++ __s32 val; ++}; ++ ++/* BTF_KIND_ARRAY is followed by one "struct btf_array" */ ++struct btf_array { ++ __u32 type; ++ __u32 index_type; ++ __u32 nelems; ++}; ++ ++/* BTF_KIND_STRUCT and BTF_KIND_UNION are followed ++ * by multiple "struct btf_member". The exact number ++ * of btf_member is stored in the vlen (of the info in ++ * "struct btf_type"). ++ */ ++struct btf_member { ++ __u32 name_off; ++ __u32 type; ++ /* If the type info kind_flag is set, the btf_member offset ++ * contains both member bitfield size and bit offset. The ++ * bitfield size is set for bitfield members. If the type ++ * info kind_flag is not set, the offset contains only bit ++ * offset. ++ */ ++ __u32 offset; ++}; ++ ++/* If the struct/union type info kind_flag is set, the ++ * following two macros are used to access bitfield_size ++ * and bit_offset from btf_member.offset. ++ */ ++#define BTF_MEMBER_BITFIELD_SIZE(val) ((val) >> 24) ++#define BTF_MEMBER_BIT_OFFSET(val) ((val) & 0xffffff) ++ ++/* BTF_KIND_FUNC_PROTO is followed by multiple "struct btf_param". ++ * The exact number of btf_param is stored in the vlen (of the ++ * info in "struct btf_type"). ++ */ ++struct btf_param { ++ __u32 name_off; ++ __u32 type; ++}; ++ ++enum { ++ BTF_VAR_STATIC = 0, ++ BTF_VAR_GLOBAL_ALLOCATED, ++}; ++ ++/* BTF_KIND_VAR is followed by a single "struct btf_var" to describe ++ * additional information related to the variable such as its linkage. ++ */ ++struct btf_var { ++ __u32 linkage; ++}; ++ ++/* BTF_KIND_DATASEC is followed by multiple "struct btf_var_secinfo" ++ * to describe all BTF_KIND_VAR types it contains along with it's ++ * in-section offset as well as size. ++ */ ++struct btf_var_secinfo { ++ __u32 type; ++ __u32 offset; ++ __u32 size; ++}; ++ ++#endif /* _UAPI__LINUX_BTF_H__ */ +--- a/kernel/bpf/arraymap.c ++++ b/kernel/bpf/arraymap.c +@@ -1,78 +1,141 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. +- * +- * This program is distributed in the hope that it will be useful, but +- * WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. ++ * Copyright (c) 2016,2017 Facebook + */ + #include ++#include + #include +-#include + #include + #include + #include + #include ++#include ++ ++#include "map_in_map.h" ++ ++#define ARRAY_CREATE_FLAG_MASK \ ++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) ++ ++static void bpf_array_free_percpu(struct bpf_array *array) ++{ ++ int i; ++ ++ for (i = 0; i < array->map.max_entries; i++) { ++ free_percpu(array->pptrs[i]); ++ cond_resched(); ++ } ++} ++ ++static int bpf_array_alloc_percpu(struct bpf_array *array) ++{ ++ void __percpu *ptr; ++ int i; ++ ++ for (i = 0; i < array->map.max_entries; i++) { ++ ptr = __alloc_percpu_gfp(array->elem_size, 8, ++ GFP_USER | __GFP_NOWARN); ++ if (!ptr) { ++ bpf_array_free_percpu(array); ++ return -ENOMEM; ++ } ++ array->pptrs[i] = ptr; ++ cond_resched(); ++ } ++ ++ return 0; ++} + + /* Called from syscall */ +-static struct bpf_map *array_map_alloc(union bpf_attr *attr) ++int array_map_alloc_check(union bpf_attr *attr) + { +- struct bpf_array *array; +- u32 elem_size, array_size; +- u32 index_mask, max_entries; +- bool unpriv = !capable(CAP_SYS_ADMIN); ++ bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; ++ int numa_node = bpf_map_attr_numa_node(attr); + + /* check sanity of attributes */ + if (attr->max_entries == 0 || attr->key_size != 4 || +- attr->value_size == 0) +- return ERR_PTR(-EINVAL); ++ attr->value_size == 0 || ++ attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || ++ !bpf_map_flags_access_ok(attr->map_flags) || ++ (percpu && numa_node != NUMA_NO_NODE)) ++ return -EINVAL; + +- if (attr->value_size >= 1 << (KMALLOC_SHIFT_MAX - 1)) ++ if (attr->value_size > KMALLOC_MAX_SIZE) + /* if value_size is bigger, the user space won't be able to + * access the elements. + */ +- return ERR_PTR(-E2BIG); ++ return -E2BIG; ++ ++ return 0; ++} ++ ++static struct bpf_map *array_map_alloc(union bpf_attr *attr) ++{ ++ bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; ++ int ret, numa_node = bpf_map_attr_numa_node(attr); ++ u32 elem_size, index_mask, max_entries; ++ bool unpriv = !capable(CAP_SYS_ADMIN); ++ u64 cost, array_size, mask64; ++ struct bpf_map_memory mem; ++ struct bpf_array *array; + + elem_size = round_up(attr->value_size, 8); + + max_entries = attr->max_entries; +- index_mask = roundup_pow_of_two(max_entries) - 1; + +- if (unpriv) ++ /* On 32 bit archs roundup_pow_of_two() with max_entries that has ++ * upper most bit set in u32 space is undefined behavior due to ++ * resulting 1U << 32, so do it manually here in u64 space. ++ */ ++ mask64 = fls_long(max_entries - 1); ++ mask64 = 1ULL << mask64; ++ mask64 -= 1; ++ ++ index_mask = mask64; ++ if (unpriv) { + /* round up array size to nearest power of 2, + * since cpu will speculate within index_mask limits + */ + max_entries = index_mask + 1; ++ /* Check for overflows. */ ++ if (max_entries < attr->max_entries) ++ return ERR_PTR(-E2BIG); ++ } + +- +- /* check round_up into zero and u32 overflow */ +- if (elem_size == 0 || +- attr->max_entries > (U32_MAX - PAGE_SIZE - sizeof(*array)) / elem_size) +- return ERR_PTR(-ENOMEM); +- +- array_size = sizeof(*array) + max_entries * elem_size; ++ array_size = sizeof(*array); ++ if (percpu) ++ array_size += (u64) max_entries * sizeof(void *); ++ else ++ array_size += (u64) max_entries * elem_size; ++ ++ /* make sure there is no u32 overflow later in round_up() */ ++ cost = array_size; ++ if (percpu) ++ cost += (u64)attr->max_entries * elem_size * num_possible_cpus(); ++ ++ ret = bpf_map_charge_init(&mem, cost); ++ if (ret < 0) ++ return ERR_PTR(ret); + + /* allocate all map elements and zero-initialize them */ +- array = kzalloc(array_size, GFP_USER | __GFP_NOWARN); ++ array = bpf_map_area_alloc(array_size, numa_node); + if (!array) { +- array = vzalloc(array_size); +- if (!array) +- return ERR_PTR(-ENOMEM); ++ bpf_map_charge_finish(&mem); ++ return ERR_PTR(-ENOMEM); + } +- + array->index_mask = index_mask; + array->map.unpriv_array = unpriv; + + /* copy mandatory map attributes */ +- array->map.key_size = attr->key_size; +- array->map.value_size = attr->value_size; +- array->map.max_entries = attr->max_entries; +- array->map.pages = round_up(array_size, PAGE_SIZE) >> PAGE_SHIFT; ++ bpf_map_init_from_attr(&array->map, attr); ++ bpf_map_charge_move(&array->map.memory, &mem); + array->elem_size = elem_size; + ++ if (percpu && bpf_array_alloc_percpu(array)) { ++ bpf_map_charge_finish(&array->map.memory); ++ bpf_map_area_free(array); ++ return ERR_PTR(-ENOMEM); ++ } ++ + return &array->map; + } + +@@ -82,17 +145,115 @@ static void *array_map_lookup_elem(struc + struct bpf_array *array = container_of(map, struct bpf_array, map); + u32 index = *(u32 *)key; + +- if (index >= array->map.max_entries) ++ if (unlikely(index >= array->map.max_entries)) + return NULL; + + return array->value + array->elem_size * (index & array->index_mask); + } + ++static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, ++ u32 off) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ ++ if (map->max_entries != 1) ++ return -ENOTSUPP; ++ if (off >= map->value_size) ++ return -EINVAL; ++ ++ *imm = (unsigned long)array->value; ++ return 0; ++} ++ ++static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, ++ u32 *off) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u64 base = (unsigned long)array->value; ++ u64 range = array->elem_size; ++ ++ if (map->max_entries != 1) ++ return -ENOTSUPP; ++ if (imm < base || imm >= base + range) ++ return -ENOENT; ++ ++ *off = imm - base; ++ return 0; ++} ++ ++/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */ ++static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ struct bpf_insn *insn = insn_buf; ++ u32 elem_size = round_up(map->value_size, 8); ++ const int ret = BPF_REG_0; ++ const int map_ptr = BPF_REG_1; ++ const int index = BPF_REG_2; ++ ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); ++ *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); ++ if (map->unpriv_array) { ++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); ++ *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); ++ } else { ++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); ++ } ++ ++ if (is_power_of_2(elem_size)) { ++ *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); ++ } else { ++ *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); ++ } ++ *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); ++ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); ++ *insn++ = BPF_MOV64_IMM(ret, 0); ++ return insn - insn_buf; ++} ++ ++/* Called from eBPF program */ ++static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u32 index = *(u32 *)key; ++ ++ if (unlikely(index >= array->map.max_entries)) ++ return NULL; ++ ++ return this_cpu_ptr(array->pptrs[index & array->index_mask]); ++} ++ ++int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u32 index = *(u32 *)key; ++ void __percpu *pptr; ++ int cpu, off = 0; ++ u32 size; ++ ++ if (unlikely(index >= array->map.max_entries)) ++ return -ENOENT; ++ ++ /* per_cpu areas are zero-filled and bpf programs can only ++ * access 'value_size' of them, so copying rounded areas ++ * will not leak any kernel data ++ */ ++ size = round_up(map->value_size, 8); ++ rcu_read_lock(); ++ pptr = array->pptrs[index & array->index_mask]; ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size); ++ off += size; ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++ + /* Called from syscall */ + static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) + { + struct bpf_array *array = container_of(map, struct bpf_array, map); +- u32 index = *(u32 *)key; ++ u32 index = key ? *(u32 *)key : U32_MAX; + u32 *next = (u32 *)next_key; + + if (index >= array->map.max_entries) { +@@ -113,22 +274,73 @@ static int array_map_update_elem(struct + { + struct bpf_array *array = container_of(map, struct bpf_array, map); + u32 index = *(u32 *)key; ++ char *val; + +- if (map_flags > BPF_EXIST) ++ if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) + /* unknown flags */ + return -EINVAL; + +- if (index >= array->map.max_entries) ++ if (unlikely(index >= array->map.max_entries)) ++ /* all elements were pre-allocated, cannot insert a new one */ ++ return -E2BIG; ++ ++ if (unlikely(map_flags & BPF_NOEXIST)) ++ /* all elements already exist */ ++ return -EEXIST; ++ ++ if (unlikely((map_flags & BPF_F_LOCK) && ++ !map_value_has_spin_lock(map))) ++ return -EINVAL; ++ ++ if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { ++ memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), ++ value, map->value_size); ++ } else { ++ val = array->value + ++ array->elem_size * (index & array->index_mask); ++ if (map_flags & BPF_F_LOCK) ++ copy_map_value_locked(map, val, value, false); ++ else ++ copy_map_value(map, val, value); ++ } ++ return 0; ++} ++ ++int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u32 index = *(u32 *)key; ++ void __percpu *pptr; ++ int cpu, off = 0; ++ u32 size; ++ ++ if (unlikely(map_flags > BPF_EXIST)) ++ /* unknown flags */ ++ return -EINVAL; ++ ++ if (unlikely(index >= array->map.max_entries)) + /* all elements were pre-allocated, cannot insert a new one */ + return -E2BIG; + +- if (map_flags == BPF_NOEXIST) ++ if (unlikely(map_flags == BPF_NOEXIST)) + /* all elements already exist */ + return -EEXIST; + +- memcpy(array->value + +- array->elem_size * (index & array->index_mask), +- value, map->value_size); ++ /* the user space will provide round_up(value_size, 8) bytes that ++ * will be copied into per-cpu area. bpf programs can only access ++ * value_size of it. During lookup the same extra bytes will be ++ * returned or zeros which were zero-filled by percpu_alloc, ++ * so no kernel data leaks possible ++ */ ++ size = round_up(map->value_size, 8); ++ rcu_read_lock(); ++ pptr = array->pptrs[index & array->index_mask]; ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size); ++ off += size; ++ } ++ rcu_read_unlock(); + return 0; + } + +@@ -150,36 +362,124 @@ static void array_map_free(struct bpf_ma + */ + synchronize_rcu(); + +- kvfree(array); ++ if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) ++ bpf_array_free_percpu(array); ++ ++ bpf_map_area_free(array); + } + +-static const struct bpf_map_ops array_ops = { ++static void array_map_seq_show_elem(struct bpf_map *map, void *key, ++ struct seq_file *m) ++{ ++ void *value; ++ ++ rcu_read_lock(); ++ ++ value = array_map_lookup_elem(map, key); ++ if (!value) { ++ rcu_read_unlock(); ++ return; ++ } ++ ++ if (map->btf_key_type_id) ++ seq_printf(m, "%u: ", *(u32 *)key); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); ++ seq_puts(m, "\n"); ++ ++ rcu_read_unlock(); ++} ++ ++static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, ++ struct seq_file *m) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u32 index = *(u32 *)key; ++ void __percpu *pptr; ++ int cpu; ++ ++ rcu_read_lock(); ++ ++ seq_printf(m, "%u: {\n", *(u32 *)key); ++ pptr = array->pptrs[index & array->index_mask]; ++ for_each_possible_cpu(cpu) { ++ seq_printf(m, "\tcpu%d: ", cpu); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, ++ per_cpu_ptr(pptr, cpu), m); ++ seq_puts(m, "\n"); ++ } ++ seq_puts(m, "}\n"); ++ ++ rcu_read_unlock(); ++} ++ ++static int array_map_check_btf(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type) ++{ ++ u32 int_data; ++ ++ /* One exception for keyless BTF: .bss/.data/.rodata map */ ++ if (btf_type_is_void(key_type)) { ++ if (map->map_type != BPF_MAP_TYPE_ARRAY || ++ map->max_entries != 1) ++ return -EINVAL; ++ ++ if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC) ++ return -EINVAL; ++ ++ return 0; ++ } ++ ++ if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) ++ return -EINVAL; ++ ++ int_data = *(u32 *)(key_type + 1); ++ /* bpf array can only take a u32 key. This check makes sure ++ * that the btf matches the attr used during map_create. ++ */ ++ if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++const struct bpf_map_ops array_map_ops = { ++ .map_alloc_check = array_map_alloc_check, + .map_alloc = array_map_alloc, + .map_free = array_map_free, + .map_get_next_key = array_map_get_next_key, + .map_lookup_elem = array_map_lookup_elem, + .map_update_elem = array_map_update_elem, + .map_delete_elem = array_map_delete_elem, ++ .map_gen_lookup = array_map_gen_lookup, ++ .map_direct_value_addr = array_map_direct_value_addr, ++ .map_direct_value_meta = array_map_direct_value_meta, ++ .map_seq_show_elem = array_map_seq_show_elem, ++ .map_check_btf = array_map_check_btf, + }; + +-static struct bpf_map_type_list array_type __read_mostly = { +- .ops = &array_ops, +- .type = BPF_MAP_TYPE_ARRAY, ++const struct bpf_map_ops percpu_array_map_ops = { ++ .map_alloc_check = array_map_alloc_check, ++ .map_alloc = array_map_alloc, ++ .map_free = array_map_free, ++ .map_get_next_key = array_map_get_next_key, ++ .map_lookup_elem = percpu_array_map_lookup_elem, ++ .map_update_elem = array_map_update_elem, ++ .map_delete_elem = array_map_delete_elem, ++ .map_seq_show_elem = percpu_array_map_seq_show_elem, ++ .map_check_btf = array_map_check_btf, + }; + +-static int __init register_array_map(void) +-{ +- bpf_register_map_type(&array_type); +- return 0; +-} +-late_initcall(register_array_map); +- +-static struct bpf_map *fd_array_map_alloc(union bpf_attr *attr) ++static int fd_array_map_alloc_check(union bpf_attr *attr) + { + /* only file descriptors can be stored in this type of map */ + if (attr->value_size != sizeof(u32)) +- return ERR_PTR(-EINVAL); +- return array_map_alloc(attr); ++ return -EINVAL; ++ /* Program read-only/write-only not supported for special maps yet. */ ++ if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) ++ return -EINVAL; ++ return array_map_alloc_check(attr); + } + + static void fd_array_map_free(struct bpf_map *map) +@@ -192,17 +492,38 @@ static void fd_array_map_free(struct bpf + /* make sure it's empty */ + for (i = 0; i < array->map.max_entries; i++) + BUG_ON(array->ptrs[i] != NULL); +- kvfree(array); ++ ++ bpf_map_area_free(array); + } + + static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) + { +- return NULL; ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++/* only called from syscall */ ++int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) ++{ ++ void **elem, *ptr; ++ int ret = 0; ++ ++ if (!map->ops->map_fd_sys_lookup_elem) ++ return -ENOTSUPP; ++ ++ rcu_read_lock(); ++ elem = array_map_lookup_elem(map, key); ++ if (elem && (ptr = READ_ONCE(*elem))) ++ *value = map->ops->map_fd_sys_lookup_elem(ptr); ++ else ++ ret = -ENOENT; ++ rcu_read_unlock(); ++ ++ return ret; + } + + /* only called from syscall */ +-static int fd_array_map_update_elem(struct bpf_map *map, void *key, +- void *value, u64 map_flags) ++int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, ++ void *key, void *value, u64 map_flags) + { + struct bpf_array *array = container_of(map, struct bpf_array, map); + void *new_ptr, *old_ptr; +@@ -215,7 +536,7 @@ static int fd_array_map_update_elem(stru + return -E2BIG; + + ufd = *(u32 *)value; +- new_ptr = map->ops->map_fd_get_ptr(map, ufd); ++ new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); + if (IS_ERR(new_ptr)) + return PTR_ERR(new_ptr); + +@@ -244,10 +565,12 @@ static int fd_array_map_delete_elem(stru + } + } + +-static void *prog_fd_array_get_ptr(struct bpf_map *map, int fd) ++static void *prog_fd_array_get_ptr(struct bpf_map *map, ++ struct file *map_file, int fd) + { + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct bpf_prog *prog = bpf_prog_get(fd); ++ + if (IS_ERR(prog)) + return prog; + +@@ -255,18 +578,22 @@ static void *prog_fd_array_get_ptr(struc + bpf_prog_put(prog); + return ERR_PTR(-EINVAL); + } ++ + return prog; + } + + static void prog_fd_array_put_ptr(void *ptr) + { +- struct bpf_prog *prog = ptr; ++ bpf_prog_put(ptr); ++} + +- bpf_prog_put_rcu(prog); ++static u32 prog_fd_array_sys_lookup_elem(void *ptr) ++{ ++ return ((struct bpf_prog *)ptr)->aux->id; + } + + /* decrement refcnt of all bpf_progs that are stored in this map */ +-void bpf_fd_array_map_clear(struct bpf_map *map) ++static void bpf_fd_array_map_clear(struct bpf_map *map) + { + struct bpf_array *array = container_of(map, struct bpf_array, map); + int i; +@@ -275,91 +602,208 @@ void bpf_fd_array_map_clear(struct bpf_m + fd_array_map_delete_elem(map, &i); + } + +-static const struct bpf_map_ops prog_array_ops = { +- .map_alloc = fd_array_map_alloc, ++static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, ++ struct seq_file *m) ++{ ++ void **elem, *ptr; ++ u32 prog_id; ++ ++ rcu_read_lock(); ++ ++ elem = array_map_lookup_elem(map, key); ++ if (elem) { ++ ptr = READ_ONCE(*elem); ++ if (ptr) { ++ seq_printf(m, "%u: ", *(u32 *)key); ++ prog_id = prog_fd_array_sys_lookup_elem(ptr); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, ++ &prog_id, m); ++ seq_puts(m, "\n"); ++ } ++ } ++ ++ rcu_read_unlock(); ++} ++ ++const struct bpf_map_ops prog_array_map_ops = { ++ .map_alloc_check = fd_array_map_alloc_check, ++ .map_alloc = array_map_alloc, + .map_free = fd_array_map_free, + .map_get_next_key = array_map_get_next_key, + .map_lookup_elem = fd_array_map_lookup_elem, +- .map_update_elem = fd_array_map_update_elem, + .map_delete_elem = fd_array_map_delete_elem, + .map_fd_get_ptr = prog_fd_array_get_ptr, + .map_fd_put_ptr = prog_fd_array_put_ptr, ++ .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem, ++ .map_release_uref = bpf_fd_array_map_clear, ++ .map_seq_show_elem = prog_array_map_seq_show_elem, + }; + +-static struct bpf_map_type_list prog_array_type __read_mostly = { +- .ops = &prog_array_ops, +- .type = BPF_MAP_TYPE_PROG_ARRAY, +-}; ++static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, ++ struct file *map_file) ++{ ++ struct bpf_event_entry *ee; ++ ++ ee = kzalloc(sizeof(*ee), GFP_ATOMIC); ++ if (ee) { ++ ee->event = perf_file->private_data; ++ ee->perf_file = perf_file; ++ ee->map_file = map_file; ++ } + +-static int __init register_prog_array_map(void) ++ return ee; ++} ++ ++static void __bpf_event_entry_free(struct rcu_head *rcu) + { +- bpf_register_map_type(&prog_array_type); +- return 0; ++ struct bpf_event_entry *ee; ++ ++ ee = container_of(rcu, struct bpf_event_entry, rcu); ++ fput(ee->perf_file); ++ kfree(ee); + } +-late_initcall(register_prog_array_map); + +-static void perf_event_array_map_free(struct bpf_map *map) ++static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee) + { +- bpf_fd_array_map_clear(map); +- fd_array_map_free(map); ++ call_rcu(&ee->rcu, __bpf_event_entry_free); + } + +-static void *perf_event_fd_array_get_ptr(struct bpf_map *map, int fd) ++static void *perf_event_fd_array_get_ptr(struct bpf_map *map, ++ struct file *map_file, int fd) + { ++ struct bpf_event_entry *ee; + struct perf_event *event; +- const struct perf_event_attr *attr; ++ struct file *perf_file; ++ u64 value; + + event = perf_event_get(fd); + if (IS_ERR(event)) + return event; + +- attr = perf_event_attrs(event); +- if (IS_ERR(attr)) +- goto err; +- +- if (attr->inherit) +- goto err; ++ value = perf_event_read_local(event); + +- if (attr->type == PERF_TYPE_RAW) +- return event; +- +- if (attr->type == PERF_TYPE_HARDWARE) +- return event; ++ ee = bpf_event_entry_gen(perf_file, map_file); ++ if (ee) ++ return ee; + +- if (attr->type == PERF_TYPE_SOFTWARE && +- attr->config == PERF_COUNT_SW_BPF_OUTPUT) +- return event; +-err: +- perf_event_release_kernel(event); +- return ERR_PTR(-EINVAL); ++ ee = ERR_PTR(-ENOMEM); ++ return ee; + } + + static void perf_event_fd_array_put_ptr(void *ptr) + { +- struct perf_event *event = ptr; ++ bpf_event_entry_free_rcu(ptr); ++} + +- perf_event_release_kernel(event); ++static void perf_event_fd_array_release(struct bpf_map *map, ++ struct file *map_file) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ struct bpf_event_entry *ee; ++ int i; ++ ++ rcu_read_lock(); ++ for (i = 0; i < array->map.max_entries; i++) { ++ ee = READ_ONCE(array->ptrs[i]); ++ if (ee && ee->map_file == map_file) ++ fd_array_map_delete_elem(map, &i); ++ } ++ rcu_read_unlock(); + } + +-static const struct bpf_map_ops perf_event_array_ops = { +- .map_alloc = fd_array_map_alloc, +- .map_free = perf_event_array_map_free, ++const struct bpf_map_ops perf_event_array_map_ops = { ++ .map_alloc_check = fd_array_map_alloc_check, ++ .map_alloc = array_map_alloc, ++ .map_free = fd_array_map_free, + .map_get_next_key = array_map_get_next_key, + .map_lookup_elem = fd_array_map_lookup_elem, +- .map_update_elem = fd_array_map_update_elem, + .map_delete_elem = fd_array_map_delete_elem, + .map_fd_get_ptr = perf_event_fd_array_get_ptr, + .map_fd_put_ptr = perf_event_fd_array_put_ptr, ++ .map_release = perf_event_fd_array_release, ++ .map_check_btf = map_check_no_btf, + }; + +-static struct bpf_map_type_list perf_event_array_type __read_mostly = { +- .ops = &perf_event_array_ops, +- .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, +-}; ++static struct bpf_map *array_of_map_alloc(union bpf_attr *attr) ++{ ++ struct bpf_map *map, *inner_map_meta; ++ ++ inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); ++ if (IS_ERR(inner_map_meta)) ++ return inner_map_meta; ++ ++ map = array_map_alloc(attr); ++ if (IS_ERR(map)) { ++ bpf_map_meta_free(inner_map_meta); ++ return map; ++ } ++ ++ map->inner_map_meta = inner_map_meta; + +-static int __init register_perf_event_array_map(void) ++ return map; ++} ++ ++static void array_of_map_free(struct bpf_map *map) + { +- bpf_register_map_type(&perf_event_array_type); +- return 0; ++ /* map->inner_map_meta is only accessed by syscall which ++ * is protected by fdget/fdput. ++ */ ++ bpf_map_meta_free(map->inner_map_meta); ++ bpf_fd_array_map_clear(map); ++ fd_array_map_free(map); + } +-late_initcall(register_perf_event_array_map); ++ ++static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_map **inner_map = array_map_lookup_elem(map, key); ++ ++ if (!inner_map) ++ return NULL; ++ ++ return READ_ONCE(*inner_map); ++} ++ ++static u32 array_of_map_gen_lookup(struct bpf_map *map, ++ struct bpf_insn *insn_buf) ++{ ++ struct bpf_array *array = container_of(map, struct bpf_array, map); ++ u32 elem_size = round_up(map->value_size, 8); ++ struct bpf_insn *insn = insn_buf; ++ const int ret = BPF_REG_0; ++ const int map_ptr = BPF_REG_1; ++ const int index = BPF_REG_2; ++ ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); ++ *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); ++ if (map->unpriv_array) { ++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); ++ *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); ++ } else { ++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); ++ } ++ if (is_power_of_2(elem_size)) ++ *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); ++ else ++ *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); ++ *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); ++ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); ++ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); ++ *insn++ = BPF_MOV64_IMM(ret, 0); ++ ++ return insn - insn_buf; ++} ++ ++const struct bpf_map_ops array_of_maps_map_ops = { ++ .map_alloc_check = fd_array_map_alloc_check, ++ .map_alloc = array_of_map_alloc, ++ .map_free = array_of_map_free, ++ .map_get_next_key = array_map_get_next_key, ++ .map_lookup_elem = array_of_map_lookup_elem, ++ .map_delete_elem = fd_array_map_delete_elem, ++ .map_fd_get_ptr = bpf_map_fd_get_ptr, ++ .map_fd_put_ptr = bpf_map_fd_put_ptr, ++ .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, ++ .map_gen_lookup = array_of_map_gen_lookup, ++ .map_check_btf = map_check_no_btf, ++}; +--- /dev/null ++++ b/kernel/bpf/bpf_lru_list.c +@@ -0,0 +1,695 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2016 Facebook ++ */ ++#include ++#include ++#include ++ ++#include "bpf_lru_list.h" ++ ++#define LOCAL_FREE_TARGET (128) ++#define LOCAL_NR_SCANS LOCAL_FREE_TARGET ++ ++#define PERCPU_FREE_TARGET (4) ++#define PERCPU_NR_SCANS PERCPU_FREE_TARGET ++ ++/* Helpers to get the local list index */ ++#define LOCAL_LIST_IDX(t) ((t) - BPF_LOCAL_LIST_T_OFFSET) ++#define LOCAL_FREE_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_FREE) ++#define LOCAL_PENDING_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_PENDING) ++#define IS_LOCAL_LIST_TYPE(t) ((t) >= BPF_LOCAL_LIST_T_OFFSET) ++ ++static int get_next_cpu(int cpu) ++{ ++ cpu = cpumask_next(cpu, cpu_possible_mask); ++ if (cpu >= nr_cpu_ids) ++ cpu = cpumask_first(cpu_possible_mask); ++ return cpu; ++} ++ ++/* Local list helpers */ ++static struct list_head *local_free_list(struct bpf_lru_locallist *loc_l) ++{ ++ return &loc_l->lists[LOCAL_FREE_LIST_IDX]; ++} ++ ++static struct list_head *local_pending_list(struct bpf_lru_locallist *loc_l) ++{ ++ return &loc_l->lists[LOCAL_PENDING_LIST_IDX]; ++} ++ ++/* bpf_lru_node helpers */ ++static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) ++{ ++ return node->ref; ++} ++ ++static void bpf_lru_list_count_inc(struct bpf_lru_list *l, ++ enum bpf_lru_list_type type) ++{ ++ if (type < NR_BPF_LRU_LIST_COUNT) ++ l->counts[type]++; ++} ++ ++static void bpf_lru_list_count_dec(struct bpf_lru_list *l, ++ enum bpf_lru_list_type type) ++{ ++ if (type < NR_BPF_LRU_LIST_COUNT) ++ l->counts[type]--; ++} ++ ++static void __bpf_lru_node_move_to_free(struct bpf_lru_list *l, ++ struct bpf_lru_node *node, ++ struct list_head *free_list, ++ enum bpf_lru_list_type tgt_free_type) ++{ ++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) ++ return; ++ ++ /* If the removing node is the next_inactive_rotation candidate, ++ * move the next_inactive_rotation pointer also. ++ */ ++ if (&node->list == l->next_inactive_rotation) ++ l->next_inactive_rotation = l->next_inactive_rotation->prev; ++ ++ bpf_lru_list_count_dec(l, node->type); ++ ++ node->type = tgt_free_type; ++ list_move(&node->list, free_list); ++} ++ ++/* Move nodes from local list to the LRU list */ ++static void __bpf_lru_node_move_in(struct bpf_lru_list *l, ++ struct bpf_lru_node *node, ++ enum bpf_lru_list_type tgt_type) ++{ ++ if (WARN_ON_ONCE(!IS_LOCAL_LIST_TYPE(node->type)) || ++ WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(tgt_type))) ++ return; ++ ++ bpf_lru_list_count_inc(l, tgt_type); ++ node->type = tgt_type; ++ node->ref = 0; ++ list_move(&node->list, &l->lists[tgt_type]); ++} ++ ++/* Move nodes between or within active and inactive list (like ++ * active to inactive, inactive to active or tail of active back to ++ * the head of active). ++ */ ++static void __bpf_lru_node_move(struct bpf_lru_list *l, ++ struct bpf_lru_node *node, ++ enum bpf_lru_list_type tgt_type) ++{ ++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)) || ++ WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(tgt_type))) ++ return; ++ ++ if (node->type != tgt_type) { ++ bpf_lru_list_count_dec(l, node->type); ++ bpf_lru_list_count_inc(l, tgt_type); ++ node->type = tgt_type; ++ } ++ node->ref = 0; ++ ++ /* If the moving node is the next_inactive_rotation candidate, ++ * move the next_inactive_rotation pointer also. ++ */ ++ if (&node->list == l->next_inactive_rotation) ++ l->next_inactive_rotation = l->next_inactive_rotation->prev; ++ ++ list_move(&node->list, &l->lists[tgt_type]); ++} ++ ++static bool bpf_lru_list_inactive_low(const struct bpf_lru_list *l) ++{ ++ return l->counts[BPF_LRU_LIST_T_INACTIVE] < ++ l->counts[BPF_LRU_LIST_T_ACTIVE]; ++} ++ ++/* Rotate the active list: ++ * 1. Start from tail ++ * 2. If the node has the ref bit set, it will be rotated ++ * back to the head of active list with the ref bit cleared. ++ * Give this node one more chance to survive in the active list. ++ * 3. If the ref bit is not set, move it to the head of the ++ * inactive list. ++ * 4. It will at most scan nr_scans nodes ++ */ ++static void __bpf_lru_list_rotate_active(struct bpf_lru *lru, ++ struct bpf_lru_list *l) ++{ ++ struct list_head *active = &l->lists[BPF_LRU_LIST_T_ACTIVE]; ++ struct bpf_lru_node *node, *tmp_node, *first_node; ++ unsigned int i = 0; ++ ++ first_node = list_first_entry(active, struct bpf_lru_node, list); ++ list_for_each_entry_safe_reverse(node, tmp_node, active, list) { ++ if (bpf_lru_node_is_ref(node)) ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); ++ else ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); ++ ++ if (++i == lru->nr_scans || node == first_node) ++ break; ++ } ++} ++ ++/* Rotate the inactive list. It starts from the next_inactive_rotation ++ * 1. If the node has ref bit set, it will be moved to the head ++ * of active list with the ref bit cleared. ++ * 2. If the node does not have ref bit set, it will leave it ++ * at its current location (i.e. do nothing) so that it can ++ * be considered during the next inactive_shrink. ++ * 3. It will at most scan nr_scans nodes ++ */ ++static void __bpf_lru_list_rotate_inactive(struct bpf_lru *lru, ++ struct bpf_lru_list *l) ++{ ++ struct list_head *inactive = &l->lists[BPF_LRU_LIST_T_INACTIVE]; ++ struct list_head *cur, *last, *next = inactive; ++ struct bpf_lru_node *node; ++ unsigned int i = 0; ++ ++ if (list_empty(inactive)) ++ return; ++ ++ last = l->next_inactive_rotation->next; ++ if (last == inactive) ++ last = last->next; ++ ++ cur = l->next_inactive_rotation; ++ while (i < lru->nr_scans) { ++ if (cur == inactive) { ++ cur = cur->prev; ++ continue; ++ } ++ ++ node = list_entry(cur, struct bpf_lru_node, list); ++ next = cur->prev; ++ if (bpf_lru_node_is_ref(node)) ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); ++ if (cur == last) ++ break; ++ cur = next; ++ i++; ++ } ++ ++ l->next_inactive_rotation = next; ++} ++ ++/* Shrink the inactive list. It starts from the tail of the ++ * inactive list and only move the nodes without the ref bit ++ * set to the designated free list. ++ */ ++static unsigned int ++__bpf_lru_list_shrink_inactive(struct bpf_lru *lru, ++ struct bpf_lru_list *l, ++ unsigned int tgt_nshrink, ++ struct list_head *free_list, ++ enum bpf_lru_list_type tgt_free_type) ++{ ++ struct list_head *inactive = &l->lists[BPF_LRU_LIST_T_INACTIVE]; ++ struct bpf_lru_node *node, *tmp_node; ++ unsigned int nshrinked = 0; ++ unsigned int i = 0; ++ ++ list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) { ++ if (bpf_lru_node_is_ref(node)) { ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); ++ } else if (lru->del_from_htab(lru->del_arg, node)) { ++ __bpf_lru_node_move_to_free(l, node, free_list, ++ tgt_free_type); ++ if (++nshrinked == tgt_nshrink) ++ break; ++ } ++ ++ if (++i == lru->nr_scans) ++ break; ++ } ++ ++ return nshrinked; ++} ++ ++/* 1. Rotate the active list (if needed) ++ * 2. Always rotate the inactive list ++ */ ++static void __bpf_lru_list_rotate(struct bpf_lru *lru, struct bpf_lru_list *l) ++{ ++ if (bpf_lru_list_inactive_low(l)) ++ __bpf_lru_list_rotate_active(lru, l); ++ ++ __bpf_lru_list_rotate_inactive(lru, l); ++} ++ ++/* Calls __bpf_lru_list_shrink_inactive() to shrink some ++ * ref-bit-cleared nodes and move them to the designated ++ * free list. ++ * ++ * If it cannot get a free node after calling ++ * __bpf_lru_list_shrink_inactive(). It will just remove ++ * one node from either inactive or active list without ++ * honoring the ref-bit. It prefers inactive list to active ++ * list in this situation. ++ */ ++static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru, ++ struct bpf_lru_list *l, ++ unsigned int tgt_nshrink, ++ struct list_head *free_list, ++ enum bpf_lru_list_type tgt_free_type) ++ ++{ ++ struct bpf_lru_node *node, *tmp_node; ++ struct list_head *force_shrink_list; ++ unsigned int nshrinked; ++ ++ nshrinked = __bpf_lru_list_shrink_inactive(lru, l, tgt_nshrink, ++ free_list, tgt_free_type); ++ if (nshrinked) ++ return nshrinked; ++ ++ /* Do a force shrink by ignoring the reference bit */ ++ if (!list_empty(&l->lists[BPF_LRU_LIST_T_INACTIVE])) ++ force_shrink_list = &l->lists[BPF_LRU_LIST_T_INACTIVE]; ++ else ++ force_shrink_list = &l->lists[BPF_LRU_LIST_T_ACTIVE]; ++ ++ list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list, ++ list) { ++ if (lru->del_from_htab(lru->del_arg, node)) { ++ __bpf_lru_node_move_to_free(l, node, free_list, ++ tgt_free_type); ++ return 1; ++ } ++ } ++ ++ return 0; ++} ++ ++/* Flush the nodes from the local pending list to the LRU list */ ++static void __local_list_flush(struct bpf_lru_list *l, ++ struct bpf_lru_locallist *loc_l) ++{ ++ struct bpf_lru_node *node, *tmp_node; ++ ++ list_for_each_entry_safe_reverse(node, tmp_node, ++ local_pending_list(loc_l), list) { ++ if (bpf_lru_node_is_ref(node)) ++ __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE); ++ else ++ __bpf_lru_node_move_in(l, node, ++ BPF_LRU_LIST_T_INACTIVE); ++ } ++} ++ ++static void bpf_lru_list_push_free(struct bpf_lru_list *l, ++ struct bpf_lru_node *node) ++{ ++ unsigned long flags; ++ ++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) ++ return; ++ ++ raw_spin_lock_irqsave(&l->lock, flags); ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); ++ raw_spin_unlock_irqrestore(&l->lock, flags); ++} ++ ++static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, ++ struct bpf_lru_locallist *loc_l) ++{ ++ struct bpf_lru_list *l = &lru->common_lru.lru_list; ++ struct bpf_lru_node *node, *tmp_node; ++ unsigned int nfree = 0; ++ ++ raw_spin_lock(&l->lock); ++ ++ __local_list_flush(l, loc_l); ++ ++ __bpf_lru_list_rotate(lru, l); ++ ++ list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE], ++ list) { ++ __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l), ++ BPF_LRU_LOCAL_LIST_T_FREE); ++ if (++nfree == LOCAL_FREE_TARGET) ++ break; ++ } ++ ++ if (nfree < LOCAL_FREE_TARGET) ++ __bpf_lru_list_shrink(lru, l, LOCAL_FREE_TARGET - nfree, ++ local_free_list(loc_l), ++ BPF_LRU_LOCAL_LIST_T_FREE); ++ ++ raw_spin_unlock(&l->lock); ++} ++ ++static void __local_list_add_pending(struct bpf_lru *lru, ++ struct bpf_lru_locallist *loc_l, ++ int cpu, ++ struct bpf_lru_node *node, ++ u32 hash) ++{ ++ *(u32 *)((void *)node + lru->hash_offset) = hash; ++ node->cpu = cpu; ++ node->type = BPF_LRU_LOCAL_LIST_T_PENDING; ++ node->ref = 0; ++ list_add(&node->list, local_pending_list(loc_l)); ++} ++ ++static struct bpf_lru_node * ++__local_list_pop_free(struct bpf_lru_locallist *loc_l) ++{ ++ struct bpf_lru_node *node; ++ ++ node = list_first_entry_or_null(local_free_list(loc_l), ++ struct bpf_lru_node, ++ list); ++ if (node) ++ list_del(&node->list); ++ ++ return node; ++} ++ ++static struct bpf_lru_node * ++__local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l) ++{ ++ struct bpf_lru_node *node; ++ bool force = false; ++ ++ignore_ref: ++ /* Get from the tail (i.e. older element) of the pending list. */ ++ list_for_each_entry_reverse(node, local_pending_list(loc_l), ++ list) { ++ if ((!bpf_lru_node_is_ref(node) || force) && ++ lru->del_from_htab(lru->del_arg, node)) { ++ list_del(&node->list); ++ return node; ++ } ++ } ++ ++ if (!force) { ++ force = true; ++ goto ignore_ref; ++ } ++ ++ return NULL; ++} ++ ++static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, ++ u32 hash) ++{ ++ struct list_head *free_list; ++ struct bpf_lru_node *node = NULL; ++ struct bpf_lru_list *l; ++ unsigned long flags; ++ int cpu = raw_smp_processor_id(); ++ ++ l = per_cpu_ptr(lru->percpu_lru, cpu); ++ ++ raw_spin_lock_irqsave(&l->lock, flags); ++ ++ __bpf_lru_list_rotate(lru, l); ++ ++ free_list = &l->lists[BPF_LRU_LIST_T_FREE]; ++ if (list_empty(free_list)) ++ __bpf_lru_list_shrink(lru, l, PERCPU_FREE_TARGET, free_list, ++ BPF_LRU_LIST_T_FREE); ++ ++ if (!list_empty(free_list)) { ++ node = list_first_entry(free_list, struct bpf_lru_node, list); ++ *(u32 *)((void *)node + lru->hash_offset) = hash; ++ node->ref = 0; ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); ++ } ++ ++ raw_spin_unlock_irqrestore(&l->lock, flags); ++ ++ return node; ++} ++ ++static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, ++ u32 hash) ++{ ++ struct bpf_lru_locallist *loc_l, *steal_loc_l; ++ struct bpf_common_lru *clru = &lru->common_lru; ++ struct bpf_lru_node *node; ++ int steal, first_steal; ++ unsigned long flags; ++ int cpu = raw_smp_processor_id(); ++ ++ loc_l = per_cpu_ptr(clru->local_list, cpu); ++ ++ raw_spin_lock_irqsave(&loc_l->lock, flags); ++ ++ node = __local_list_pop_free(loc_l); ++ if (!node) { ++ bpf_lru_list_pop_free_to_local(lru, loc_l); ++ node = __local_list_pop_free(loc_l); ++ } ++ ++ if (node) ++ __local_list_add_pending(lru, loc_l, cpu, node, hash); ++ ++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); ++ ++ if (node) ++ return node; ++ ++ /* No free nodes found from the local free list and ++ * the global LRU list. ++ * ++ * Steal from the local free/pending list of the ++ * current CPU and remote CPU in RR. It starts ++ * with the loc_l->next_steal CPU. ++ */ ++ ++ first_steal = loc_l->next_steal; ++ steal = first_steal; ++ do { ++ steal_loc_l = per_cpu_ptr(clru->local_list, steal); ++ ++ raw_spin_lock_irqsave(&steal_loc_l->lock, flags); ++ ++ node = __local_list_pop_free(steal_loc_l); ++ if (!node) ++ node = __local_list_pop_pending(lru, steal_loc_l); ++ ++ raw_spin_unlock_irqrestore(&steal_loc_l->lock, flags); ++ ++ steal = get_next_cpu(steal); ++ } while (!node && steal != first_steal); ++ ++ loc_l->next_steal = steal; ++ ++ if (node) { ++ raw_spin_lock_irqsave(&loc_l->lock, flags); ++ __local_list_add_pending(lru, loc_l, cpu, node, hash); ++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); ++ } ++ ++ return node; ++} ++ ++struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) ++{ ++ if (lru->percpu) ++ return bpf_percpu_lru_pop_free(lru, hash); ++ else ++ return bpf_common_lru_pop_free(lru, hash); ++} ++ ++static void bpf_common_lru_push_free(struct bpf_lru *lru, ++ struct bpf_lru_node *node) ++{ ++ u8 node_type = READ_ONCE(node->type); ++ unsigned long flags; ++ ++ if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) || ++ WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE)) ++ return; ++ ++ if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) { ++ struct bpf_lru_locallist *loc_l; ++ ++ loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); ++ ++ raw_spin_lock_irqsave(&loc_l->lock, flags); ++ ++ if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { ++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); ++ goto check_lru_list; ++ } ++ ++ node->type = BPF_LRU_LOCAL_LIST_T_FREE; ++ node->ref = 0; ++ list_move(&node->list, local_free_list(loc_l)); ++ ++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); ++ return; ++ } ++ ++check_lru_list: ++ bpf_lru_list_push_free(&lru->common_lru.lru_list, node); ++} ++ ++static void bpf_percpu_lru_push_free(struct bpf_lru *lru, ++ struct bpf_lru_node *node) ++{ ++ struct bpf_lru_list *l; ++ unsigned long flags; ++ ++ l = per_cpu_ptr(lru->percpu_lru, node->cpu); ++ ++ raw_spin_lock_irqsave(&l->lock, flags); ++ ++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); ++ ++ raw_spin_unlock_irqrestore(&l->lock, flags); ++} ++ ++void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node) ++{ ++ if (lru->percpu) ++ bpf_percpu_lru_push_free(lru, node); ++ else ++ bpf_common_lru_push_free(lru, node); ++} ++ ++static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, ++ u32 node_offset, u32 elem_size, ++ u32 nr_elems) ++{ ++ struct bpf_lru_list *l = &lru->common_lru.lru_list; ++ u32 i; ++ ++ for (i = 0; i < nr_elems; i++) { ++ struct bpf_lru_node *node; ++ ++ node = (struct bpf_lru_node *)(buf + node_offset); ++ node->type = BPF_LRU_LIST_T_FREE; ++ node->ref = 0; ++ list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); ++ buf += elem_size; ++ } ++} ++ ++static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf, ++ u32 node_offset, u32 elem_size, ++ u32 nr_elems) ++{ ++ u32 i, pcpu_entries; ++ int cpu; ++ struct bpf_lru_list *l; ++ ++ pcpu_entries = nr_elems / num_possible_cpus(); ++ ++ i = 0; ++ ++ for_each_possible_cpu(cpu) { ++ struct bpf_lru_node *node; ++ ++ l = per_cpu_ptr(lru->percpu_lru, cpu); ++again: ++ node = (struct bpf_lru_node *)(buf + node_offset); ++ node->cpu = cpu; ++ node->type = BPF_LRU_LIST_T_FREE; ++ node->ref = 0; ++ list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); ++ i++; ++ buf += elem_size; ++ if (i == nr_elems) ++ break; ++ if (i % pcpu_entries) ++ goto again; ++ } ++} ++ ++void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, ++ u32 elem_size, u32 nr_elems) ++{ ++ if (lru->percpu) ++ bpf_percpu_lru_populate(lru, buf, node_offset, elem_size, ++ nr_elems); ++ else ++ bpf_common_lru_populate(lru, buf, node_offset, elem_size, ++ nr_elems); ++} ++ ++static void bpf_lru_locallist_init(struct bpf_lru_locallist *loc_l, int cpu) ++{ ++ int i; ++ ++ for (i = 0; i < NR_BPF_LRU_LOCAL_LIST_T; i++) ++ INIT_LIST_HEAD(&loc_l->lists[i]); ++ ++ loc_l->next_steal = cpu; ++ ++ raw_spin_lock_init(&loc_l->lock); ++} ++ ++static void bpf_lru_list_init(struct bpf_lru_list *l) ++{ ++ int i; ++ ++ for (i = 0; i < NR_BPF_LRU_LIST_T; i++) ++ INIT_LIST_HEAD(&l->lists[i]); ++ ++ for (i = 0; i < NR_BPF_LRU_LIST_COUNT; i++) ++ l->counts[i] = 0; ++ ++ l->next_inactive_rotation = &l->lists[BPF_LRU_LIST_T_INACTIVE]; ++ ++ raw_spin_lock_init(&l->lock); ++} ++ ++int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, ++ del_from_htab_func del_from_htab, void *del_arg) ++{ ++ int cpu; ++ ++ if (percpu) { ++ lru->percpu_lru = alloc_percpu(struct bpf_lru_list); ++ if (!lru->percpu_lru) ++ return -ENOMEM; ++ ++ for_each_possible_cpu(cpu) { ++ struct bpf_lru_list *l; ++ ++ l = per_cpu_ptr(lru->percpu_lru, cpu); ++ bpf_lru_list_init(l); ++ } ++ lru->nr_scans = PERCPU_NR_SCANS; ++ } else { ++ struct bpf_common_lru *clru = &lru->common_lru; ++ ++ clru->local_list = alloc_percpu(struct bpf_lru_locallist); ++ if (!clru->local_list) ++ return -ENOMEM; ++ ++ for_each_possible_cpu(cpu) { ++ struct bpf_lru_locallist *loc_l; ++ ++ loc_l = per_cpu_ptr(clru->local_list, cpu); ++ bpf_lru_locallist_init(loc_l, cpu); ++ } ++ ++ bpf_lru_list_init(&clru->lru_list); ++ lru->nr_scans = LOCAL_NR_SCANS; ++ } ++ ++ lru->percpu = percpu; ++ lru->del_from_htab = del_from_htab; ++ lru->del_arg = del_arg; ++ lru->hash_offset = hash_offset; ++ ++ return 0; ++} ++ ++void bpf_lru_destroy(struct bpf_lru *lru) ++{ ++ if (lru->percpu) ++ free_percpu(lru->percpu_lru); ++ else ++ free_percpu(lru->common_lru.local_list); ++} +--- /dev/null ++++ b/kernel/bpf/bpf_lru_list.h +@@ -0,0 +1,82 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright (c) 2016 Facebook ++ */ ++#ifndef __BPF_LRU_LIST_H_ ++#define __BPF_LRU_LIST_H_ ++ ++#include ++#include ++ ++#define NR_BPF_LRU_LIST_T (3) ++#define NR_BPF_LRU_LIST_COUNT (2) ++#define NR_BPF_LRU_LOCAL_LIST_T (2) ++#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T ++ ++enum bpf_lru_list_type { ++ BPF_LRU_LIST_T_ACTIVE, ++ BPF_LRU_LIST_T_INACTIVE, ++ BPF_LRU_LIST_T_FREE, ++ BPF_LRU_LOCAL_LIST_T_FREE, ++ BPF_LRU_LOCAL_LIST_T_PENDING, ++}; ++ ++struct bpf_lru_node { ++ struct list_head list; ++ u16 cpu; ++ u8 type; ++ u8 ref; ++}; ++ ++struct bpf_lru_list { ++ struct list_head lists[NR_BPF_LRU_LIST_T]; ++ unsigned int counts[NR_BPF_LRU_LIST_COUNT]; ++ /* The next inacitve list rotation starts from here */ ++ struct list_head *next_inactive_rotation; ++ ++ raw_spinlock_t lock ____cacheline_aligned_in_smp; ++}; ++ ++struct bpf_lru_locallist { ++ struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T]; ++ u16 next_steal; ++ raw_spinlock_t lock; ++}; ++ ++struct bpf_common_lru { ++ struct bpf_lru_list lru_list; ++ struct bpf_lru_locallist __percpu *local_list; ++}; ++ ++typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node); ++ ++struct bpf_lru { ++ union { ++ struct bpf_common_lru common_lru; ++ struct bpf_lru_list __percpu *percpu_lru; ++ }; ++ del_from_htab_func del_from_htab; ++ void *del_arg; ++ unsigned int hash_offset; ++ unsigned int nr_scans; ++ bool percpu; ++}; ++ ++static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node) ++{ ++ /* ref is an approximation on access frequency. It does not ++ * have to be very accurate. Hence, no protection is used. ++ */ ++ if (!node->ref) ++ node->ref = 1; ++} ++ ++int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, ++ del_from_htab_func del_from_htab, void *delete_arg); ++void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, ++ u32 elem_size, u32 nr_elems); ++void bpf_lru_destroy(struct bpf_lru *lru); ++struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash); ++void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node); ++void bpf_lru_promote(struct bpf_lru *lru, struct bpf_lru_node *node); ++ ++#endif +--- /dev/null ++++ b/kernel/bpf/btf.c +@@ -0,0 +1,3514 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* Copyright (c) 2018 Facebook */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* BTF (BPF Type Format) is the meta data format which describes ++ * the data types of BPF program/map. Hence, it basically focus ++ * on the C programming language which the modern BPF is primary ++ * using. ++ * ++ * ELF Section: ++ * ~~~~~~~~~~~ ++ * The BTF data is stored under the ".BTF" ELF section ++ * ++ * struct btf_type: ++ * ~~~~~~~~~~~~~~~ ++ * Each 'struct btf_type' object describes a C data type. ++ * Depending on the type it is describing, a 'struct btf_type' ++ * object may be followed by more data. F.e. ++ * To describe an array, 'struct btf_type' is followed by ++ * 'struct btf_array'. ++ * ++ * 'struct btf_type' and any extra data following it are ++ * 4 bytes aligned. ++ * ++ * Type section: ++ * ~~~~~~~~~~~~~ ++ * The BTF type section contains a list of 'struct btf_type' objects. ++ * Each one describes a C type. Recall from the above section ++ * that a 'struct btf_type' object could be immediately followed by extra ++ * data in order to desribe some particular C types. ++ * ++ * type_id: ++ * ~~~~~~~ ++ * Each btf_type object is identified by a type_id. The type_id ++ * is implicitly implied by the location of the btf_type object in ++ * the BTF type section. The first one has type_id 1. The second ++ * one has type_id 2...etc. Hence, an earlier btf_type has ++ * a smaller type_id. ++ * ++ * A btf_type object may refer to another btf_type object by using ++ * type_id (i.e. the "type" in the "struct btf_type"). ++ * ++ * NOTE that we cannot assume any reference-order. ++ * A btf_type object can refer to an earlier btf_type object ++ * but it can also refer to a later btf_type object. ++ * ++ * For example, to describe "const void *". A btf_type ++ * object describing "const" may refer to another btf_type ++ * object describing "void *". This type-reference is done ++ * by specifying type_id: ++ * ++ * [1] CONST (anon) type_id=2 ++ * [2] PTR (anon) type_id=0 ++ * ++ * The above is the btf_verifier debug log: ++ * - Each line started with "[?]" is a btf_type object ++ * - [?] is the type_id of the btf_type object. ++ * - CONST/PTR is the BTF_KIND_XXX ++ * - "(anon)" is the name of the type. It just ++ * happens that CONST and PTR has no name. ++ * - type_id=XXX is the 'u32 type' in btf_type ++ * ++ * NOTE: "void" has type_id 0 ++ * ++ * String section: ++ * ~~~~~~~~~~~~~~ ++ * The BTF string section contains the names used by the type section. ++ * Each string is referred by an "offset" from the beginning of the ++ * string section. ++ * ++ * Each string is '\0' terminated. ++ * ++ * The first character in the string section must be '\0' ++ * which is used to mean 'anonymous'. Some btf_type may not ++ * have a name. ++ */ ++ ++/* BTF verification: ++ * ++ * To verify BTF data, two passes are needed. ++ * ++ * Pass #1 ++ * ~~~~~~~ ++ * The first pass is to collect all btf_type objects to ++ * an array: "btf->types". ++ * ++ * Depending on the C type that a btf_type is describing, ++ * a btf_type may be followed by extra data. We don't know ++ * how many btf_type is there, and more importantly we don't ++ * know where each btf_type is located in the type section. ++ * ++ * Without knowing the location of each type_id, most verifications ++ * cannot be done. e.g. an earlier btf_type may refer to a later ++ * btf_type (recall the "const void *" above), so we cannot ++ * check this type-reference in the first pass. ++ * ++ * In the first pass, it still does some verifications (e.g. ++ * checking the name is a valid offset to the string section). ++ * ++ * Pass #2 ++ * ~~~~~~~ ++ * The main focus is to resolve a btf_type that is referring ++ * to another type. ++ * ++ * We have to ensure the referring type: ++ * 1) does exist in the BTF (i.e. in btf->types[]) ++ * 2) does not cause a loop: ++ * struct A { ++ * struct B b; ++ * }; ++ * ++ * struct B { ++ * struct A a; ++ * }; ++ * ++ * btf_type_needs_resolve() decides if a btf_type needs ++ * to be resolved. ++ * ++ * The needs_resolve type implements the "resolve()" ops which ++ * essentially does a DFS and detects backedge. ++ * ++ * During resolve (or DFS), different C types have different ++ * "RESOLVED" conditions. ++ * ++ * When resolving a BTF_KIND_STRUCT, we need to resolve all its ++ * members because a member is always referring to another ++ * type. A struct's member can be treated as "RESOLVED" if ++ * it is referring to a BTF_KIND_PTR. Otherwise, the ++ * following valid C struct would be rejected: ++ * ++ * struct A { ++ * int m; ++ * struct A *a; ++ * }; ++ * ++ * When resolving a BTF_KIND_PTR, it needs to keep resolving if ++ * it is referring to another BTF_KIND_PTR. Otherwise, we cannot ++ * detect a pointer loop, e.g.: ++ * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR + ++ * ^ | ++ * +-----------------------------------------+ ++ * ++ */ ++ ++#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2) ++#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) ++#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK) ++#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3) ++#define BITS_ROUNDUP_BYTES(bits) \ ++ (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) ++ ++#define BTF_INFO_MASK 0x8f00ffff ++#define BTF_INT_MASK 0x0fffffff ++#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) ++#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) ++ ++/* 16MB for 64k structs and each has 16 members and ++ * a few MB spaces for the string section. ++ * The hard limit is S32_MAX. ++ */ ++#define BTF_MAX_SIZE (16 * 1024 * 1024) ++ ++#define for_each_member(i, struct_type, member) \ ++ for (i = 0, member = btf_type_member(struct_type); \ ++ i < btf_type_vlen(struct_type); \ ++ i++, member++) ++ ++#define for_each_member_from(i, from, struct_type, member) \ ++ for (i = from, member = btf_type_member(struct_type) + from; \ ++ i < btf_type_vlen(struct_type); \ ++ i++, member++) ++ ++#define for_each_vsi(i, struct_type, member) \ ++ for (i = 0, member = btf_type_var_secinfo(struct_type); \ ++ i < btf_type_vlen(struct_type); \ ++ i++, member++) ++ ++#define for_each_vsi_from(i, from, struct_type, member) \ ++ for (i = from, member = btf_type_var_secinfo(struct_type) + from; \ ++ i < btf_type_vlen(struct_type); \ ++ i++, member++) ++ ++DEFINE_IDR(btf_idr); ++DEFINE_SPINLOCK(btf_idr_lock); ++ ++struct btf { ++ void *data; ++ struct btf_type **types; ++ u32 *resolved_ids; ++ u32 *resolved_sizes; ++ const char *strings; ++ void *nohdr_data; ++ struct btf_header hdr; ++ u32 nr_types; ++ u32 types_size; ++ u32 data_size; ++ refcount_t refcnt; ++ u32 id; ++ struct rcu_head rcu; ++}; ++ ++enum verifier_phase { ++ CHECK_META, ++ CHECK_TYPE, ++}; ++ ++struct resolve_vertex { ++ const struct btf_type *t; ++ u32 type_id; ++ u16 next_member; ++}; ++ ++enum visit_state { ++ NOT_VISITED, ++ VISITED, ++ RESOLVED, ++}; ++ ++enum resolve_mode { ++ RESOLVE_TBD, /* To Be Determined */ ++ RESOLVE_PTR, /* Resolving for Pointer */ ++ RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union ++ * or array ++ */ ++}; ++ ++#define MAX_RESOLVE_DEPTH 32 ++ ++struct btf_sec_info { ++ u32 off; ++ u32 len; ++}; ++ ++struct btf_verifier_env { ++ struct btf *btf; ++ u8 *visit_states; ++ struct resolve_vertex stack[MAX_RESOLVE_DEPTH]; ++ struct bpf_verifier_log log; ++ u32 log_type_id; ++ u32 top_stack; ++ enum verifier_phase phase; ++ enum resolve_mode resolve_mode; ++}; ++ ++static const char * const btf_kind_str[NR_BTF_KINDS] = { ++ [BTF_KIND_UNKN] = "UNKNOWN", ++ [BTF_KIND_INT] = "INT", ++ [BTF_KIND_PTR] = "PTR", ++ [BTF_KIND_ARRAY] = "ARRAY", ++ [BTF_KIND_STRUCT] = "STRUCT", ++ [BTF_KIND_UNION] = "UNION", ++ [BTF_KIND_ENUM] = "ENUM", ++ [BTF_KIND_FWD] = "FWD", ++ [BTF_KIND_TYPEDEF] = "TYPEDEF", ++ [BTF_KIND_VOLATILE] = "VOLATILE", ++ [BTF_KIND_CONST] = "CONST", ++ [BTF_KIND_RESTRICT] = "RESTRICT", ++ [BTF_KIND_FUNC] = "FUNC", ++ [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", ++ [BTF_KIND_VAR] = "VAR", ++ [BTF_KIND_DATASEC] = "DATASEC", ++}; ++ ++struct btf_kind_operations { ++ s32 (*check_meta)(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left); ++ int (*resolve)(struct btf_verifier_env *env, ++ const struct resolve_vertex *v); ++ int (*check_member)(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type); ++ int (*check_kflag_member)(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type); ++ void (*log_details)(struct btf_verifier_env *env, ++ const struct btf_type *t); ++ void (*seq_show)(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offsets, ++ struct seq_file *m); ++}; ++ ++static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS]; ++static struct btf_type btf_void; ++ ++static int btf_resolve(struct btf_verifier_env *env, ++ const struct btf_type *t, u32 type_id); ++ ++static bool btf_type_is_modifier(const struct btf_type *t) ++{ ++ /* Some of them is not strictly a C modifier ++ * but they are grouped into the same bucket ++ * for BTF concern: ++ * A type (t) that refers to another ++ * type through t->type AND its size cannot ++ * be determined without following the t->type. ++ * ++ * ptr does not fall into this bucket ++ * because its size is always sizeof(void *). ++ */ ++ switch (BTF_INFO_KIND(t->info)) { ++ case BTF_KIND_TYPEDEF: ++ case BTF_KIND_VOLATILE: ++ case BTF_KIND_CONST: ++ case BTF_KIND_RESTRICT: ++ return true; ++ } ++ ++ return false; ++} ++ ++bool btf_type_is_void(const struct btf_type *t) ++{ ++ return t == &btf_void; ++} ++ ++static bool btf_type_is_fwd(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; ++} ++ ++static bool btf_type_is_func(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC; ++} ++ ++static bool btf_type_is_func_proto(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO; ++} ++ ++static bool btf_type_nosize(const struct btf_type *t) ++{ ++ return btf_type_is_void(t) || btf_type_is_fwd(t) || ++ btf_type_is_func(t) || btf_type_is_func_proto(t); ++} ++ ++static bool btf_type_nosize_or_null(const struct btf_type *t) ++{ ++ return !t || btf_type_nosize(t); ++} ++ ++/* union is only a special case of struct: ++ * all its offsetof(member) == 0 ++ */ ++static bool btf_type_is_struct(const struct btf_type *t) ++{ ++ u8 kind = BTF_INFO_KIND(t->info); ++ ++ return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; ++} ++ ++static bool __btf_type_is_struct(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT; ++} ++ ++static bool btf_type_is_array(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; ++} ++ ++static bool btf_type_is_ptr(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; ++} ++ ++static bool btf_type_is_int(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_INT; ++} ++ ++static bool btf_type_is_var(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_VAR; ++} ++ ++static bool btf_type_is_datasec(const struct btf_type *t) ++{ ++ return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; ++} ++ ++/* Types that act only as a source, not sink or intermediate ++ * type when resolving. ++ */ ++static bool btf_type_is_resolve_source_only(const struct btf_type *t) ++{ ++ return btf_type_is_var(t) || ++ btf_type_is_datasec(t); ++} ++ ++/* What types need to be resolved? ++ * ++ * btf_type_is_modifier() is an obvious one. ++ * ++ * btf_type_is_struct() because its member refers to ++ * another type (through member->type). ++ * ++ * btf_type_is_var() because the variable refers to ++ * another type. btf_type_is_datasec() holds multiple ++ * btf_type_is_var() types that need resolving. ++ * ++ * btf_type_is_array() because its element (array->type) ++ * refers to another type. Array can be thought of a ++ * special case of struct while array just has the same ++ * member-type repeated by array->nelems of times. ++ */ ++static bool btf_type_needs_resolve(const struct btf_type *t) ++{ ++ return btf_type_is_modifier(t) || ++ btf_type_is_ptr(t) || ++ btf_type_is_struct(t) || ++ btf_type_is_array(t) || ++ btf_type_is_var(t) || ++ btf_type_is_datasec(t); ++} ++ ++/* t->size can be used */ ++static bool btf_type_has_size(const struct btf_type *t) ++{ ++ switch (BTF_INFO_KIND(t->info)) { ++ case BTF_KIND_INT: ++ case BTF_KIND_STRUCT: ++ case BTF_KIND_UNION: ++ case BTF_KIND_ENUM: ++ case BTF_KIND_DATASEC: ++ return true; ++ } ++ ++ return false; ++} ++ ++static const char *btf_int_encoding_str(u8 encoding) ++{ ++ if (encoding == 0) ++ return "(none)"; ++ else if (encoding == BTF_INT_SIGNED) ++ return "SIGNED"; ++ else if (encoding == BTF_INT_CHAR) ++ return "CHAR"; ++ else if (encoding == BTF_INT_BOOL) ++ return "BOOL"; ++ else ++ return "UNKN"; ++} ++ ++static u16 btf_type_vlen(const struct btf_type *t) ++{ ++ return BTF_INFO_VLEN(t->info); ++} ++ ++static bool btf_type_kflag(const struct btf_type *t) ++{ ++ return BTF_INFO_KFLAG(t->info); ++} ++ ++static u32 btf_member_bit_offset(const struct btf_type *struct_type, ++ const struct btf_member *member) ++{ ++ return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset) ++ : member->offset; ++} ++ ++static u32 btf_member_bitfield_size(const struct btf_type *struct_type, ++ const struct btf_member *member) ++{ ++ return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset) ++ : 0; ++} ++ ++static u32 btf_type_int(const struct btf_type *t) ++{ ++ return *(u32 *)(t + 1); ++} ++ ++static const struct btf_array *btf_type_array(const struct btf_type *t) ++{ ++ return (const struct btf_array *)(t + 1); ++} ++ ++static const struct btf_member *btf_type_member(const struct btf_type *t) ++{ ++ return (const struct btf_member *)(t + 1); ++} ++ ++static const struct btf_enum *btf_type_enum(const struct btf_type *t) ++{ ++ return (const struct btf_enum *)(t + 1); ++} ++ ++static const struct btf_var *btf_type_var(const struct btf_type *t) ++{ ++ return (const struct btf_var *)(t + 1); ++} ++ ++static const struct btf_var_secinfo *btf_type_var_secinfo(const struct btf_type *t) ++{ ++ return (const struct btf_var_secinfo *)(t + 1); ++} ++ ++static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) ++{ ++ return kind_ops[BTF_INFO_KIND(t->info)]; ++} ++ ++static bool btf_name_offset_valid(const struct btf *btf, u32 offset) ++{ ++ return BTF_STR_OFFSET_VALID(offset) && ++ offset < btf->hdr.str_len; ++} ++ ++static bool __btf_name_char_ok(char c, bool first, bool dot_ok) ++{ ++ if ((first ? !isalpha(c) : ++ !isalnum(c)) && ++ c != '_' && ++ ((c == '.' && !dot_ok) || ++ c != '.')) ++ return false; ++ return true; ++} ++ ++static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok) ++{ ++ /* offset must be valid */ ++ const char *src = &btf->strings[offset]; ++ const char *src_limit; ++ ++ if (!__btf_name_char_ok(*src, true, dot_ok)) ++ return false; ++ ++ /* set a limit on identifier length */ ++ src_limit = src + KSYM_NAME_LEN; ++ src++; ++ while (*src && src < src_limit) { ++ if (!__btf_name_char_ok(*src, false, dot_ok)) ++ return false; ++ src++; ++ } ++ ++ return !*src; ++} ++ ++/* Only C-style identifier is permitted. This can be relaxed if ++ * necessary. ++ */ ++static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) ++{ ++ return __btf_name_valid(btf, offset, false); ++} ++ ++static bool btf_name_valid_section(const struct btf *btf, u32 offset) ++{ ++ return __btf_name_valid(btf, offset, true); ++} ++ ++static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) ++{ ++ if (!offset) ++ return "(anon)"; ++ else if (offset < btf->hdr.str_len) ++ return &btf->strings[offset]; ++ else ++ return "(invalid-name-offset)"; ++} ++ ++const char *btf_name_by_offset(const struct btf *btf, u32 offset) ++{ ++ if (offset < btf->hdr.str_len) ++ return &btf->strings[offset]; ++ ++ return NULL; ++} ++ ++const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) ++{ ++ if (type_id > btf->nr_types) ++ return NULL; ++ ++ return btf->types[type_id]; ++} ++ ++/* ++ * Regular int is not a bit field and it must be either ++ * u8/u16/u32/u64 or __int128. ++ */ ++static bool btf_type_int_is_regular(const struct btf_type *t) ++{ ++ u8 nr_bits, nr_bytes; ++ u32 int_data; ++ ++ int_data = btf_type_int(t); ++ nr_bits = BTF_INT_BITS(int_data); ++ nr_bytes = BITS_ROUNDUP_BYTES(nr_bits); ++ if (BITS_PER_BYTE_MASKED(nr_bits) || ++ BTF_INT_OFFSET(int_data) || ++ (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) && ++ nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) && ++ nr_bytes != (2 * sizeof(u64)))) { ++ return false; ++ } ++ ++ return true; ++} ++ ++/* ++ * Check that given struct member is a regular int with expected ++ * offset and size. ++ */ ++bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, ++ const struct btf_member *m, ++ u32 expected_offset, u32 expected_size) ++{ ++ const struct btf_type *t; ++ u32 id, int_data; ++ u8 nr_bits; ++ ++ id = m->type; ++ t = btf_type_id_size(btf, &id, NULL); ++ if (!t || !btf_type_is_int(t)) ++ return false; ++ ++ int_data = btf_type_int(t); ++ nr_bits = BTF_INT_BITS(int_data); ++ if (btf_type_kflag(s)) { ++ u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); ++ u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); ++ ++ /* if kflag set, int should be a regular int and ++ * bit offset should be at byte boundary. ++ */ ++ return !bitfield_size && ++ BITS_ROUNDUP_BYTES(bit_offset) == expected_offset && ++ BITS_ROUNDUP_BYTES(nr_bits) == expected_size; ++ } ++ ++ if (BTF_INT_OFFSET(int_data) || ++ BITS_PER_BYTE_MASKED(m->offset) || ++ BITS_ROUNDUP_BYTES(m->offset) != expected_offset || ++ BITS_PER_BYTE_MASKED(nr_bits) || ++ BITS_ROUNDUP_BYTES(nr_bits) != expected_size) ++ return false; ++ ++ return true; ++} ++ ++__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log, ++ const char *fmt, ...) ++{ ++ va_list args; ++ ++ va_start(args, fmt); ++ bpf_verifier_vlog(log, fmt, args); ++ va_end(args); ++} ++ ++__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env, ++ const char *fmt, ...) ++{ ++ struct bpf_verifier_log *log = &env->log; ++ va_list args; ++ ++ if (!bpf_verifier_log_needed(log)) ++ return; ++ ++ va_start(args, fmt); ++ bpf_verifier_vlog(log, fmt, args); ++ va_end(args); ++} ++ ++__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ bool log_details, ++ const char *fmt, ...) ++{ ++ struct bpf_verifier_log *log = &env->log; ++ u8 kind = BTF_INFO_KIND(t->info); ++ struct btf *btf = env->btf; ++ va_list args; ++ ++ if (!bpf_verifier_log_needed(log)) ++ return; ++ ++ __btf_verifier_log(log, "[%u] %s %s%s", ++ env->log_type_id, ++ btf_kind_str[kind], ++ __btf_name_by_offset(btf, t->name_off), ++ log_details ? " " : ""); ++ ++ if (log_details) ++ btf_type_ops(t)->log_details(env, t); ++ ++ if (fmt && *fmt) { ++ __btf_verifier_log(log, " "); ++ va_start(args, fmt); ++ bpf_verifier_vlog(log, fmt, args); ++ va_end(args); ++ } ++ ++ __btf_verifier_log(log, "\n"); ++} ++ ++#define btf_verifier_log_type(env, t, ...) \ ++ __btf_verifier_log_type((env), (t), true, __VA_ARGS__) ++#define btf_verifier_log_basic(env, t, ...) \ ++ __btf_verifier_log_type((env), (t), false, __VA_ARGS__) ++ ++__printf(4, 5) ++static void btf_verifier_log_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const char *fmt, ...) ++{ ++ struct bpf_verifier_log *log = &env->log; ++ struct btf *btf = env->btf; ++ va_list args; ++ ++ if (!bpf_verifier_log_needed(log)) ++ return; ++ ++ /* The CHECK_META phase already did a btf dump. ++ * ++ * If member is logged again, it must hit an error in ++ * parsing this member. It is useful to print out which ++ * struct this member belongs to. ++ */ ++ if (env->phase != CHECK_META) ++ btf_verifier_log_type(env, struct_type, NULL); ++ ++ if (btf_type_kflag(struct_type)) ++ __btf_verifier_log(log, ++ "\t%s type_id=%u bitfield_size=%u bits_offset=%u", ++ __btf_name_by_offset(btf, member->name_off), ++ member->type, ++ BTF_MEMBER_BITFIELD_SIZE(member->offset), ++ BTF_MEMBER_BIT_OFFSET(member->offset)); ++ else ++ __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u", ++ __btf_name_by_offset(btf, member->name_off), ++ member->type, member->offset); ++ ++ if (fmt && *fmt) { ++ __btf_verifier_log(log, " "); ++ va_start(args, fmt); ++ bpf_verifier_vlog(log, fmt, args); ++ va_end(args); ++ } ++ ++ __btf_verifier_log(log, "\n"); ++} ++ ++__printf(4, 5) ++static void btf_verifier_log_vsi(struct btf_verifier_env *env, ++ const struct btf_type *datasec_type, ++ const struct btf_var_secinfo *vsi, ++ const char *fmt, ...) ++{ ++ struct bpf_verifier_log *log = &env->log; ++ va_list args; ++ ++ if (!bpf_verifier_log_needed(log)) ++ return; ++ if (env->phase != CHECK_META) ++ btf_verifier_log_type(env, datasec_type, NULL); ++ ++ __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u", ++ vsi->type, vsi->offset, vsi->size); ++ if (fmt && *fmt) { ++ __btf_verifier_log(log, " "); ++ va_start(args, fmt); ++ bpf_verifier_vlog(log, fmt, args); ++ va_end(args); ++ } ++ ++ __btf_verifier_log(log, "\n"); ++} ++ ++static void btf_verifier_log_hdr(struct btf_verifier_env *env, ++ u32 btf_data_size) ++{ ++ struct bpf_verifier_log *log = &env->log; ++ const struct btf *btf = env->btf; ++ const struct btf_header *hdr; ++ ++ if (!bpf_verifier_log_needed(log)) ++ return; ++ ++ hdr = &btf->hdr; ++ __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); ++ __btf_verifier_log(log, "version: %u\n", hdr->version); ++ __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); ++ __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); ++ __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); ++ __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); ++ __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); ++ __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); ++ __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size); ++} ++ ++static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) ++{ ++ struct btf *btf = env->btf; ++ ++ /* < 2 because +1 for btf_void which is always in btf->types[0]. ++ * btf_void is not accounted in btf->nr_types because btf_void ++ * does not come from the BTF file. ++ */ ++ if (btf->types_size - btf->nr_types < 2) { ++ /* Expand 'types' array */ ++ ++ struct btf_type **new_types; ++ u32 expand_by, new_size; ++ ++ if (btf->types_size == BTF_MAX_TYPE) { ++ btf_verifier_log(env, "Exceeded max num of types"); ++ return -E2BIG; ++ } ++ ++ expand_by = max_t(u32, btf->types_size >> 2, 16); ++ new_size = min_t(u32, BTF_MAX_TYPE, ++ btf->types_size + expand_by); ++ ++ new_types = kcalloc(new_size, sizeof(*new_types), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!new_types) ++ return -ENOMEM; ++ ++ if (btf->nr_types == 0) ++ new_types[0] = &btf_void; ++ else ++ memcpy(new_types, btf->types, ++ sizeof(*btf->types) * (btf->nr_types + 1)); ++ ++ kvfree(btf->types); ++ btf->types = new_types; ++ btf->types_size = new_size; ++ } ++ ++ btf->types[++(btf->nr_types)] = t; ++ ++ return 0; ++} ++ ++static int btf_alloc_id(struct btf *btf) ++{ ++ int id; ++ ++ idr_preload(GFP_KERNEL); ++ spin_lock_bh(&btf_idr_lock); ++ id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); ++ if (id > 0) ++ btf->id = id; ++ spin_unlock_bh(&btf_idr_lock); ++ idr_preload_end(); ++ ++ if (WARN_ON_ONCE(!id)) ++ return -ENOSPC; ++ ++ return id > 0 ? 0 : id; ++} ++ ++static void btf_free_id(struct btf *btf) ++{ ++ unsigned long flags; ++ ++ /* ++ * In map-in-map, calling map_delete_elem() on outer ++ * map will call bpf_map_put on the inner map. ++ * It will then eventually call btf_free_id() ++ * on the inner map. Some of the map_delete_elem() ++ * implementation may have irq disabled, so ++ * we need to use the _irqsave() version instead ++ * of the _bh() version. ++ */ ++ spin_lock_irqsave(&btf_idr_lock, flags); ++ idr_remove(&btf_idr, btf->id); ++ spin_unlock_irqrestore(&btf_idr_lock, flags); ++} ++ ++static void btf_free(struct btf *btf) ++{ ++ kvfree(btf->types); ++ kvfree(btf->resolved_sizes); ++ kvfree(btf->resolved_ids); ++ kvfree(btf->data); ++ kfree(btf); ++} ++ ++static void btf_free_rcu(struct rcu_head *rcu) ++{ ++ struct btf *btf = container_of(rcu, struct btf, rcu); ++ ++ btf_free(btf); ++} ++ ++void btf_put(struct btf *btf) ++{ ++ if (btf && refcount_dec_and_test(&btf->refcnt)) { ++ btf_free_id(btf); ++ call_rcu(&btf->rcu, btf_free_rcu); ++ } ++} ++ ++static int env_resolve_init(struct btf_verifier_env *env) ++{ ++ struct btf *btf = env->btf; ++ u32 nr_types = btf->nr_types; ++ u32 *resolved_sizes = NULL; ++ u32 *resolved_ids = NULL; ++ u8 *visit_states = NULL; ++ ++ /* +1 for btf_void */ ++ resolved_sizes = kcalloc(nr_types + 1, sizeof(*resolved_sizes), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!resolved_sizes) ++ goto nomem; ++ ++ resolved_ids = kcalloc(nr_types + 1, sizeof(*resolved_ids), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!resolved_ids) ++ goto nomem; ++ ++ visit_states = kcalloc(nr_types + 1, sizeof(*visit_states), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!visit_states) ++ goto nomem; ++ ++ btf->resolved_sizes = resolved_sizes; ++ btf->resolved_ids = resolved_ids; ++ env->visit_states = visit_states; ++ ++ return 0; ++ ++nomem: ++ kvfree(resolved_sizes); ++ kvfree(resolved_ids); ++ kvfree(visit_states); ++ return -ENOMEM; ++} ++ ++static void btf_verifier_env_free(struct btf_verifier_env *env) ++{ ++ kvfree(env->visit_states); ++ kfree(env); ++} ++ ++static bool env_type_is_resolve_sink(const struct btf_verifier_env *env, ++ const struct btf_type *next_type) ++{ ++ switch (env->resolve_mode) { ++ case RESOLVE_TBD: ++ /* int, enum or void is a sink */ ++ return !btf_type_needs_resolve(next_type); ++ case RESOLVE_PTR: ++ /* int, enum, void, struct, array, func or func_proto is a sink ++ * for ptr ++ */ ++ return !btf_type_is_modifier(next_type) && ++ !btf_type_is_ptr(next_type); ++ case RESOLVE_STRUCT_OR_ARRAY: ++ /* int, enum, void, ptr, func or func_proto is a sink ++ * for struct and array ++ */ ++ return !btf_type_is_modifier(next_type) && ++ !btf_type_is_array(next_type) && ++ !btf_type_is_struct(next_type); ++ default: ++ BUG(); ++ } ++} ++ ++static bool env_type_is_resolved(const struct btf_verifier_env *env, ++ u32 type_id) ++{ ++ return env->visit_states[type_id] == RESOLVED; ++} ++ ++static int env_stack_push(struct btf_verifier_env *env, ++ const struct btf_type *t, u32 type_id) ++{ ++ struct resolve_vertex *v; ++ ++ if (env->top_stack == MAX_RESOLVE_DEPTH) ++ return -E2BIG; ++ ++ if (env->visit_states[type_id] != NOT_VISITED) ++ return -EEXIST; ++ ++ env->visit_states[type_id] = VISITED; ++ ++ v = &env->stack[env->top_stack++]; ++ v->t = t; ++ v->type_id = type_id; ++ v->next_member = 0; ++ ++ if (env->resolve_mode == RESOLVE_TBD) { ++ if (btf_type_is_ptr(t)) ++ env->resolve_mode = RESOLVE_PTR; ++ else if (btf_type_is_struct(t) || btf_type_is_array(t)) ++ env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; ++ } ++ ++ return 0; ++} ++ ++static void env_stack_set_next_member(struct btf_verifier_env *env, ++ u16 next_member) ++{ ++ env->stack[env->top_stack - 1].next_member = next_member; ++} ++ ++static void env_stack_pop_resolved(struct btf_verifier_env *env, ++ u32 resolved_type_id, ++ u32 resolved_size) ++{ ++ u32 type_id = env->stack[--(env->top_stack)].type_id; ++ struct btf *btf = env->btf; ++ ++ btf->resolved_sizes[type_id] = resolved_size; ++ btf->resolved_ids[type_id] = resolved_type_id; ++ env->visit_states[type_id] = RESOLVED; ++} ++ ++static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) ++{ ++ return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; ++} ++ ++/* The input param "type_id" must point to a needs_resolve type */ ++static const struct btf_type *btf_type_id_resolve(const struct btf *btf, ++ u32 *type_id) ++{ ++ *type_id = btf->resolved_ids[*type_id]; ++ return btf_type_by_id(btf, *type_id); ++} ++ ++const struct btf_type *btf_type_id_size(const struct btf *btf, ++ u32 *type_id, u32 *ret_size) ++{ ++ const struct btf_type *size_type; ++ u32 size_type_id = *type_id; ++ u32 size = 0; ++ ++ size_type = btf_type_by_id(btf, size_type_id); ++ if (btf_type_nosize_or_null(size_type)) ++ return NULL; ++ ++ if (btf_type_has_size(size_type)) { ++ size = size_type->size; ++ } else if (btf_type_is_array(size_type)) { ++ size = btf->resolved_sizes[size_type_id]; ++ } else if (btf_type_is_ptr(size_type)) { ++ size = sizeof(void *); ++ } else { ++ if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) && ++ !btf_type_is_var(size_type))) ++ return NULL; ++ ++ size_type_id = btf->resolved_ids[size_type_id]; ++ size_type = btf_type_by_id(btf, size_type_id); ++ if (btf_type_nosize_or_null(size_type)) ++ return NULL; ++ else if (btf_type_has_size(size_type)) ++ size = size_type->size; ++ else if (btf_type_is_array(size_type)) ++ size = btf->resolved_sizes[size_type_id]; ++ else if (btf_type_is_ptr(size_type)) ++ size = sizeof(void *); ++ else ++ return NULL; ++ } ++ ++ *type_id = size_type_id; ++ if (ret_size) ++ *ret_size = size; ++ ++ return size_type; ++} ++ ++static int btf_df_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ btf_verifier_log_basic(env, struct_type, ++ "Unsupported check_member"); ++ return -EINVAL; ++} ++ ++static int btf_df_check_kflag_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ btf_verifier_log_basic(env, struct_type, ++ "Unsupported check_kflag_member"); ++ return -EINVAL; ++} ++ ++/* Used for ptr, array and struct/union type members. ++ * int, enum and modifier types have their specific callback functions. ++ */ ++static int btf_generic_check_kflag_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member bitfield_size"); ++ return -EINVAL; ++ } ++ ++ /* bitfield size is 0, so member->offset represents bit offset only. ++ * It is safe to call non kflag check_member variants. ++ */ ++ return btf_type_ops(member_type)->check_member(env, struct_type, ++ member, ++ member_type); ++} ++ ++static int btf_df_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ btf_verifier_log_basic(env, v->t, "Unsupported resolve"); ++ return -EINVAL; ++} ++ ++static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offsets, ++ struct seq_file *m) ++{ ++ seq_printf(m, "", BTF_INFO_KIND(t->info)); ++} ++ ++static int btf_int_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 int_data = btf_type_int(member_type); ++ u32 struct_bits_off = member->offset; ++ u32 struct_size = struct_type->size; ++ u32 nr_copy_bits; ++ u32 bytes_offset; ++ ++ if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "bits_offset exceeds U32_MAX"); ++ return -EINVAL; ++ } ++ ++ struct_bits_off += BTF_INT_OFFSET(int_data); ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ nr_copy_bits = BTF_INT_BITS(int_data) + ++ BITS_PER_BYTE_MASKED(struct_bits_off); ++ ++ if (nr_copy_bits > BITS_PER_U128) { ++ btf_verifier_log_member(env, struct_type, member, ++ "nr_copy_bits exceeds 128"); ++ return -EINVAL; ++ } ++ ++ if (struct_size < bytes_offset || ++ struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int btf_int_check_kflag_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset; ++ u32 int_data = btf_type_int(member_type); ++ u32 struct_size = struct_type->size; ++ u32 nr_copy_bits; ++ ++ /* a regular int type is required for the kflag int member */ ++ if (!btf_type_int_is_regular(member_type)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member base type"); ++ return -EINVAL; ++ } ++ ++ /* check sanity of bitfield size */ ++ nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); ++ struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); ++ nr_int_data_bits = BTF_INT_BITS(int_data); ++ if (!nr_bits) { ++ /* Not a bitfield member, member offset must be at byte ++ * boundary. ++ */ ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member offset"); ++ return -EINVAL; ++ } ++ ++ nr_bits = nr_int_data_bits; ++ } else if (nr_bits > nr_int_data_bits) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member bitfield_size"); ++ return -EINVAL; ++ } ++ ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off); ++ if (nr_copy_bits > BITS_PER_U128) { ++ btf_verifier_log_member(env, struct_type, member, ++ "nr_copy_bits exceeds 128"); ++ return -EINVAL; ++ } ++ ++ if (struct_size < bytes_offset || ++ struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static s32 btf_int_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ u32 int_data, nr_bits, meta_needed = sizeof(int_data); ++ u16 encoding; ++ ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ int_data = btf_type_int(t); ++ if (int_data & ~BTF_INT_MASK) { ++ btf_verifier_log_basic(env, t, "Invalid int_data:%x", ++ int_data); ++ return -EINVAL; ++ } ++ ++ nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data); ++ ++ if (nr_bits > BITS_PER_U128) { ++ btf_verifier_log_type(env, t, "nr_bits exceeds %zu", ++ BITS_PER_U128); ++ return -EINVAL; ++ } ++ ++ if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { ++ btf_verifier_log_type(env, t, "nr_bits exceeds type_size"); ++ return -EINVAL; ++ } ++ ++ /* ++ * Only one of the encoding bits is allowed and it ++ * should be sufficient for the pretty print purpose (i.e. decoding). ++ * Multiple bits can be allowed later if it is found ++ * to be insufficient. ++ */ ++ encoding = BTF_INT_ENCODING(int_data); ++ if (encoding && ++ encoding != BTF_INT_SIGNED && ++ encoding != BTF_INT_CHAR && ++ encoding != BTF_INT_BOOL) { ++ btf_verifier_log_type(env, t, "Unsupported encoding"); ++ return -ENOTSUPP; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return meta_needed; ++} ++ ++static void btf_int_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ int int_data = btf_type_int(t); ++ ++ btf_verifier_log(env, ++ "size=%u bits_offset=%u nr_bits=%u encoding=%s", ++ t->size, BTF_INT_OFFSET(int_data), ++ BTF_INT_BITS(int_data), ++ btf_int_encoding_str(BTF_INT_ENCODING(int_data))); ++} ++ ++static void btf_int128_print(struct seq_file *m, void *data) ++{ ++ /* data points to a __int128 number. ++ * Suppose ++ * int128_num = *(__int128 *)data; ++ * The below formulas shows what upper_num and lower_num represents: ++ * upper_num = int128_num >> 64; ++ * lower_num = int128_num & 0xffffffffFFFFFFFFULL; ++ */ ++ u64 upper_num, lower_num; ++ ++#ifdef __BIG_ENDIAN_BITFIELD ++ upper_num = *(u64 *)data; ++ lower_num = *(u64 *)(data + 8); ++#else ++ upper_num = *(u64 *)(data + 8); ++ lower_num = *(u64 *)data; ++#endif ++ if (upper_num == 0) ++ seq_printf(m, "0x%llx", lower_num); ++ else ++ seq_printf(m, "0x%llx%016llx", upper_num, lower_num); ++} ++ ++static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, ++ u16 right_shift_bits) ++{ ++ u64 upper_num, lower_num; ++ ++#ifdef __BIG_ENDIAN_BITFIELD ++ upper_num = print_num[0]; ++ lower_num = print_num[1]; ++#else ++ upper_num = print_num[1]; ++ lower_num = print_num[0]; ++#endif ++ ++ /* shake out un-needed bits by shift/or operations */ ++ if (left_shift_bits >= 64) { ++ upper_num = lower_num << (left_shift_bits - 64); ++ lower_num = 0; ++ } else { ++ upper_num = (upper_num << left_shift_bits) | ++ (lower_num >> (64 - left_shift_bits)); ++ lower_num = lower_num << left_shift_bits; ++ } ++ ++ if (right_shift_bits >= 64) { ++ lower_num = upper_num >> (right_shift_bits - 64); ++ upper_num = 0; ++ } else { ++ lower_num = (lower_num >> right_shift_bits) | ++ (upper_num << (64 - right_shift_bits)); ++ upper_num = upper_num >> right_shift_bits; ++ } ++ ++#ifdef __BIG_ENDIAN_BITFIELD ++ print_num[0] = upper_num; ++ print_num[1] = lower_num; ++#else ++ print_num[0] = lower_num; ++ print_num[1] = upper_num; ++#endif ++} ++ ++static void btf_bitfield_seq_show(void *data, u8 bits_offset, ++ u8 nr_bits, struct seq_file *m) ++{ ++ u16 left_shift_bits, right_shift_bits; ++ u8 nr_copy_bytes; ++ u8 nr_copy_bits; ++ u64 print_num[2] = {}; ++ ++ nr_copy_bits = nr_bits + bits_offset; ++ nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits); ++ ++ memcpy(print_num, data, nr_copy_bytes); ++ ++#ifdef __BIG_ENDIAN_BITFIELD ++ left_shift_bits = bits_offset; ++#else ++ left_shift_bits = BITS_PER_U128 - nr_copy_bits; ++#endif ++ right_shift_bits = BITS_PER_U128 - nr_bits; ++ ++ btf_int128_shift(print_num, left_shift_bits, right_shift_bits); ++ btf_int128_print(m, print_num); ++} ++ ++ ++static void btf_int_bits_seq_show(const struct btf *btf, ++ const struct btf_type *t, ++ void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ u32 int_data = btf_type_int(t); ++ u8 nr_bits = BTF_INT_BITS(int_data); ++ u8 total_bits_offset; ++ ++ /* ++ * bits_offset is at most 7. ++ * BTF_INT_OFFSET() cannot exceed 128 bits. ++ */ ++ total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data); ++ data += BITS_ROUNDDOWN_BYTES(total_bits_offset); ++ bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset); ++ btf_bitfield_seq_show(data, bits_offset, nr_bits, m); ++} ++ ++static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ u32 int_data = btf_type_int(t); ++ u8 encoding = BTF_INT_ENCODING(int_data); ++ bool sign = encoding & BTF_INT_SIGNED; ++ u8 nr_bits = BTF_INT_BITS(int_data); ++ ++ if (bits_offset || BTF_INT_OFFSET(int_data) || ++ BITS_PER_BYTE_MASKED(nr_bits)) { ++ btf_int_bits_seq_show(btf, t, data, bits_offset, m); ++ return; ++ } ++ ++ switch (nr_bits) { ++ case 128: ++ btf_int128_print(m, data); ++ break; ++ case 64: ++ if (sign) ++ seq_printf(m, "%lld", *(s64 *)data); ++ else ++ seq_printf(m, "%llu", *(u64 *)data); ++ break; ++ case 32: ++ if (sign) ++ seq_printf(m, "%d", *(s32 *)data); ++ else ++ seq_printf(m, "%u", *(u32 *)data); ++ break; ++ case 16: ++ if (sign) ++ seq_printf(m, "%d", *(s16 *)data); ++ else ++ seq_printf(m, "%u", *(u16 *)data); ++ break; ++ case 8: ++ if (sign) ++ seq_printf(m, "%d", *(s8 *)data); ++ else ++ seq_printf(m, "%u", *(u8 *)data); ++ break; ++ default: ++ btf_int_bits_seq_show(btf, t, data, bits_offset, m); ++ } ++} ++ ++static const struct btf_kind_operations int_ops = { ++ .check_meta = btf_int_check_meta, ++ .resolve = btf_df_resolve, ++ .check_member = btf_int_check_member, ++ .check_kflag_member = btf_int_check_kflag_member, ++ .log_details = btf_int_log, ++ .seq_show = btf_int_seq_show, ++}; ++ ++static int btf_modifier_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ const struct btf_type *resolved_type; ++ u32 resolved_type_id = member->type; ++ struct btf_member resolved_member; ++ struct btf *btf = env->btf; ++ ++ resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); ++ if (!resolved_type) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member"); ++ return -EINVAL; ++ } ++ ++ resolved_member = *member; ++ resolved_member.type = resolved_type_id; ++ ++ return btf_type_ops(resolved_type)->check_member(env, struct_type, ++ &resolved_member, ++ resolved_type); ++} ++ ++static int btf_modifier_check_kflag_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ const struct btf_type *resolved_type; ++ u32 resolved_type_id = member->type; ++ struct btf_member resolved_member; ++ struct btf *btf = env->btf; ++ ++ resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); ++ if (!resolved_type) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member"); ++ return -EINVAL; ++ } ++ ++ resolved_member = *member; ++ resolved_member.type = resolved_type_id; ++ ++ return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, ++ &resolved_member, ++ resolved_type); ++} ++ ++static int btf_ptr_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_size, struct_bits_off, bytes_offset; ++ ++ struct_size = struct_type->size; ++ struct_bits_off = member->offset; ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member is not byte aligned"); ++ return -EINVAL; ++ } ++ ++ if (struct_size - bytes_offset < sizeof(void *)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int btf_ref_type_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ if (!BTF_TYPE_ID_VALID(t->type)) { ++ btf_verifier_log_type(env, t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ /* typedef type must have a valid name, and other ref types, ++ * volatile, const, restrict, should have a null name. ++ */ ++ if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { ++ if (!t->name_off || ++ !btf_name_valid_identifier(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ } else { ++ if (t->name_off) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return 0; ++} ++ ++static int btf_modifier_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_type *t = v->t; ++ const struct btf_type *next_type; ++ u32 next_type_id = t->type; ++ struct btf *btf = env->btf; ++ ++ next_type = btf_type_by_id(btf, next_type_id); ++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, next_type) && ++ !env_type_is_resolved(env, next_type_id)) ++ return env_stack_push(env, next_type, next_type_id); ++ ++ /* Figure out the resolved next_type_id with size. ++ * They will be stored in the current modifier's ++ * resolved_ids and resolved_sizes such that it can ++ * save us a few type-following when we use it later (e.g. in ++ * pretty print). ++ */ ++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { ++ if (env_type_is_resolved(env, next_type_id)) ++ next_type = btf_type_id_resolve(btf, &next_type_id); ++ ++ /* "typedef void new_void", "const void"...etc */ ++ if (!btf_type_is_void(next_type) && ++ !btf_type_is_fwd(next_type) && ++ !btf_type_is_func_proto(next_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ } ++ ++ env_stack_pop_resolved(env, next_type_id, 0); ++ ++ return 0; ++} ++ ++static int btf_var_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_type *next_type; ++ const struct btf_type *t = v->t; ++ u32 next_type_id = t->type; ++ struct btf *btf = env->btf; ++ ++ next_type = btf_type_by_id(btf, next_type_id); ++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, next_type) && ++ !env_type_is_resolved(env, next_type_id)) ++ return env_stack_push(env, next_type, next_type_id); ++ ++ if (btf_type_is_modifier(next_type)) { ++ const struct btf_type *resolved_type; ++ u32 resolved_type_id; ++ ++ resolved_type_id = next_type_id; ++ resolved_type = btf_type_id_resolve(btf, &resolved_type_id); ++ ++ if (btf_type_is_ptr(resolved_type) && ++ !env_type_is_resolve_sink(env, resolved_type) && ++ !env_type_is_resolved(env, resolved_type_id)) ++ return env_stack_push(env, resolved_type, ++ resolved_type_id); ++ } ++ ++ /* We must resolve to something concrete at this point, no ++ * forward types or similar that would resolve to size of ++ * zero is allowed. ++ */ ++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ env_stack_pop_resolved(env, next_type_id, 0); ++ ++ return 0; ++} ++ ++static int btf_ptr_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_type *next_type; ++ const struct btf_type *t = v->t; ++ u32 next_type_id = t->type; ++ struct btf *btf = env->btf; ++ ++ next_type = btf_type_by_id(btf, next_type_id); ++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, next_type) && ++ !env_type_is_resolved(env, next_type_id)) ++ return env_stack_push(env, next_type, next_type_id); ++ ++ /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY, ++ * the modifier may have stopped resolving when it was resolved ++ * to a ptr (last-resolved-ptr). ++ * ++ * We now need to continue from the last-resolved-ptr to ++ * ensure the last-resolved-ptr will not referring back to ++ * the currenct ptr (t). ++ */ ++ if (btf_type_is_modifier(next_type)) { ++ const struct btf_type *resolved_type; ++ u32 resolved_type_id; ++ ++ resolved_type_id = next_type_id; ++ resolved_type = btf_type_id_resolve(btf, &resolved_type_id); ++ ++ if (btf_type_is_ptr(resolved_type) && ++ !env_type_is_resolve_sink(env, resolved_type) && ++ !env_type_is_resolved(env, resolved_type_id)) ++ return env_stack_push(env, resolved_type, ++ resolved_type_id); ++ } ++ ++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { ++ if (env_type_is_resolved(env, next_type_id)) ++ next_type = btf_type_id_resolve(btf, &next_type_id); ++ ++ if (!btf_type_is_void(next_type) && ++ !btf_type_is_fwd(next_type) && ++ !btf_type_is_func_proto(next_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ } ++ ++ env_stack_pop_resolved(env, next_type_id, 0); ++ ++ return 0; ++} ++ ++static void btf_modifier_seq_show(const struct btf *btf, ++ const struct btf_type *t, ++ u32 type_id, void *data, ++ u8 bits_offset, struct seq_file *m) ++{ ++ t = btf_type_id_resolve(btf, &type_id); ++ ++ btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); ++} ++ ++static void btf_var_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ t = btf_type_id_resolve(btf, &type_id); ++ ++ btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); ++} ++ ++static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ /* It is a hashed value */ ++ seq_printf(m, "%p", *(void **)data); ++} ++ ++static void btf_ref_type_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ btf_verifier_log(env, "type_id=%u", t->type); ++} ++ ++static struct btf_kind_operations modifier_ops = { ++ .check_meta = btf_ref_type_check_meta, ++ .resolve = btf_modifier_resolve, ++ .check_member = btf_modifier_check_member, ++ .check_kflag_member = btf_modifier_check_kflag_member, ++ .log_details = btf_ref_type_log, ++ .seq_show = btf_modifier_seq_show, ++}; ++ ++static struct btf_kind_operations ptr_ops = { ++ .check_meta = btf_ref_type_check_meta, ++ .resolve = btf_ptr_resolve, ++ .check_member = btf_ptr_check_member, ++ .check_kflag_member = btf_generic_check_kflag_member, ++ .log_details = btf_ref_type_log, ++ .seq_show = btf_ptr_seq_show, ++}; ++ ++static s32 btf_fwd_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (t->type) { ++ btf_verifier_log_type(env, t, "type != 0"); ++ return -EINVAL; ++ } ++ ++ /* fwd type must have a valid name */ ++ if (!t->name_off || ++ !btf_name_valid_identifier(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return 0; ++} ++ ++static void btf_fwd_type_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct"); ++} ++ ++static struct btf_kind_operations fwd_ops = { ++ .check_meta = btf_fwd_check_meta, ++ .resolve = btf_df_resolve, ++ .check_member = btf_df_check_member, ++ .check_kflag_member = btf_df_check_kflag_member, ++ .log_details = btf_fwd_type_log, ++ .seq_show = btf_df_seq_show, ++}; ++ ++static int btf_array_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_bits_off = member->offset; ++ u32 struct_size, bytes_offset; ++ u32 array_type_id, array_size; ++ struct btf *btf = env->btf; ++ ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member is not byte aligned"); ++ return -EINVAL; ++ } ++ ++ array_type_id = member->type; ++ btf_type_id_size(btf, &array_type_id, &array_size); ++ struct_size = struct_type->size; ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ if (struct_size - bytes_offset < array_size) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static s32 btf_array_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ const struct btf_array *array = btf_type_array(t); ++ u32 meta_needed = sizeof(*array); ++ ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ /* array type should not have a name */ ++ if (t->name_off) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ if (t->size) { ++ btf_verifier_log_type(env, t, "size != 0"); ++ return -EINVAL; ++ } ++ ++ /* Array elem type and index type cannot be in type void, ++ * so !array->type and !array->index_type are not allowed. ++ */ ++ if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { ++ btf_verifier_log_type(env, t, "Invalid elem"); ++ return -EINVAL; ++ } ++ ++ if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { ++ btf_verifier_log_type(env, t, "Invalid index"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return meta_needed; ++} ++ ++static int btf_array_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_array *array = btf_type_array(v->t); ++ const struct btf_type *elem_type, *index_type; ++ u32 elem_type_id, index_type_id; ++ struct btf *btf = env->btf; ++ u32 elem_size; ++ ++ /* Check array->index_type */ ++ index_type_id = array->index_type; ++ index_type = btf_type_by_id(btf, index_type_id); ++ if (btf_type_nosize_or_null(index_type) || ++ btf_type_is_resolve_source_only(index_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid index"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, index_type) && ++ !env_type_is_resolved(env, index_type_id)) ++ return env_stack_push(env, index_type, index_type_id); ++ ++ index_type = btf_type_id_size(btf, &index_type_id, NULL); ++ if (!index_type || !btf_type_is_int(index_type) || ++ !btf_type_int_is_regular(index_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid index"); ++ return -EINVAL; ++ } ++ ++ /* Check array->type */ ++ elem_type_id = array->type; ++ elem_type = btf_type_by_id(btf, elem_type_id); ++ if (btf_type_nosize_or_null(elem_type) || ++ btf_type_is_resolve_source_only(elem_type)) { ++ btf_verifier_log_type(env, v->t, ++ "Invalid elem"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, elem_type) && ++ !env_type_is_resolved(env, elem_type_id)) ++ return env_stack_push(env, elem_type, elem_type_id); ++ ++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); ++ if (!elem_type) { ++ btf_verifier_log_type(env, v->t, "Invalid elem"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) { ++ btf_verifier_log_type(env, v->t, "Invalid array of int"); ++ return -EINVAL; ++ } ++ ++ if (array->nelems && elem_size > U32_MAX / array->nelems) { ++ btf_verifier_log_type(env, v->t, ++ "Array size overflows U32_MAX"); ++ return -EINVAL; ++ } ++ ++ env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); ++ ++ return 0; ++} ++ ++static void btf_array_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ const struct btf_array *array = btf_type_array(t); ++ ++ btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u", ++ array->type, array->index_type, array->nelems); ++} ++ ++static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ const struct btf_array *array = btf_type_array(t); ++ const struct btf_kind_operations *elem_ops; ++ const struct btf_type *elem_type; ++ u32 i, elem_size, elem_type_id; ++ ++ elem_type_id = array->type; ++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); ++ elem_ops = btf_type_ops(elem_type); ++ seq_puts(m, "["); ++ for (i = 0; i < array->nelems; i++) { ++ if (i) ++ seq_puts(m, ","); ++ ++ elem_ops->seq_show(btf, elem_type, elem_type_id, data, ++ bits_offset, m); ++ data += elem_size; ++ } ++ seq_puts(m, "]"); ++} ++ ++static struct btf_kind_operations array_ops = { ++ .check_meta = btf_array_check_meta, ++ .resolve = btf_array_resolve, ++ .check_member = btf_array_check_member, ++ .check_kflag_member = btf_generic_check_kflag_member, ++ .log_details = btf_array_log, ++ .seq_show = btf_array_seq_show, ++}; ++ ++static int btf_struct_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_bits_off = member->offset; ++ u32 struct_size, bytes_offset; ++ ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member is not byte aligned"); ++ return -EINVAL; ++ } ++ ++ struct_size = struct_type->size; ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ if (struct_size - bytes_offset < member_type->size) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static s32 btf_struct_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; ++ const struct btf_member *member; ++ u32 meta_needed, last_offset; ++ struct btf *btf = env->btf; ++ u32 struct_size = t->size; ++ u32 offset; ++ u16 i; ++ ++ meta_needed = btf_type_vlen(t) * sizeof(*member); ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ /* struct type either no name or a valid one */ ++ if (t->name_off && ++ !btf_name_valid_identifier(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ last_offset = 0; ++ for_each_member(i, t, member) { ++ if (!btf_name_offset_valid(btf, member->name_off)) { ++ btf_verifier_log_member(env, t, member, ++ "Invalid member name_offset:%u", ++ member->name_off); ++ return -EINVAL; ++ } ++ ++ /* struct member either no name or a valid one */ ++ if (member->name_off && ++ !btf_name_valid_identifier(btf, member->name_off)) { ++ btf_verifier_log_member(env, t, member, "Invalid name"); ++ return -EINVAL; ++ } ++ /* A member cannot be in type void */ ++ if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { ++ btf_verifier_log_member(env, t, member, ++ "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ offset = btf_member_bit_offset(t, member); ++ if (is_union && offset) { ++ btf_verifier_log_member(env, t, member, ++ "Invalid member bits_offset"); ++ return -EINVAL; ++ } ++ ++ /* ++ * ">" instead of ">=" because the last member could be ++ * "char a[0];" ++ */ ++ if (last_offset > offset) { ++ btf_verifier_log_member(env, t, member, ++ "Invalid member bits_offset"); ++ return -EINVAL; ++ } ++ ++ if (BITS_ROUNDUP_BYTES(offset) > struct_size) { ++ btf_verifier_log_member(env, t, member, ++ "Member bits_offset exceeds its struct size"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_member(env, t, member, NULL); ++ last_offset = offset; ++ } ++ ++ return meta_needed; ++} ++ ++static int btf_struct_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_member *member; ++ int err; ++ u16 i; ++ ++ /* Before continue resolving the next_member, ++ * ensure the last member is indeed resolved to a ++ * type with size info. ++ */ ++ if (v->next_member) { ++ const struct btf_type *last_member_type; ++ const struct btf_member *last_member; ++ u16 last_member_type_id; ++ ++ last_member = btf_type_member(v->t) + v->next_member - 1; ++ last_member_type_id = last_member->type; ++ if (WARN_ON_ONCE(!env_type_is_resolved(env, ++ last_member_type_id))) ++ return -EINVAL; ++ ++ last_member_type = btf_type_by_id(env->btf, ++ last_member_type_id); ++ if (btf_type_kflag(v->t)) ++ err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, ++ last_member, ++ last_member_type); ++ else ++ err = btf_type_ops(last_member_type)->check_member(env, v->t, ++ last_member, ++ last_member_type); ++ if (err) ++ return err; ++ } ++ ++ for_each_member_from(i, v->next_member, v->t, member) { ++ u32 member_type_id = member->type; ++ const struct btf_type *member_type = btf_type_by_id(env->btf, ++ member_type_id); ++ ++ if (btf_type_nosize_or_null(member_type) || ++ btf_type_is_resolve_source_only(member_type)) { ++ btf_verifier_log_member(env, v->t, member, ++ "Invalid member"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, member_type) && ++ !env_type_is_resolved(env, member_type_id)) { ++ env_stack_set_next_member(env, i + 1); ++ return env_stack_push(env, member_type, member_type_id); ++ } ++ ++ if (btf_type_kflag(v->t)) ++ err = btf_type_ops(member_type)->check_kflag_member(env, v->t, ++ member, ++ member_type); ++ else ++ err = btf_type_ops(member_type)->check_member(env, v->t, ++ member, ++ member_type); ++ if (err) ++ return err; ++ } ++ ++ env_stack_pop_resolved(env, 0, 0); ++ ++ return 0; ++} ++ ++static void btf_struct_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); ++} ++ ++/* find 'struct bpf_spin_lock' in map value. ++ * return >= 0 offset if found ++ * and < 0 in case of error ++ */ ++int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t) ++{ ++ const struct btf_member *member; ++ u32 i, off = -ENOENT; ++ ++ if (!__btf_type_is_struct(t)) ++ return -EINVAL; ++ ++ for_each_member(i, t, member) { ++ const struct btf_type *member_type = btf_type_by_id(btf, ++ member->type); ++ if (!__btf_type_is_struct(member_type)) ++ continue; ++ if (member_type->size != sizeof(struct bpf_spin_lock)) ++ continue; ++ if (strcmp(__btf_name_by_offset(btf, member_type->name_off), ++ "bpf_spin_lock")) ++ continue; ++ if (off != -ENOENT) ++ /* only one 'struct bpf_spin_lock' is allowed */ ++ return -E2BIG; ++ off = btf_member_bit_offset(t, member); ++ if (off % 8) ++ /* valid C code cannot generate such BTF */ ++ return -EINVAL; ++ off /= 8; ++ if (off % __alignof__(struct bpf_spin_lock)) ++ /* valid struct bpf_spin_lock will be 4 byte aligned */ ++ return -EINVAL; ++ } ++ return off; ++} ++ ++static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ","; ++ const struct btf_member *member; ++ u32 i; ++ ++ seq_puts(m, "{"); ++ for_each_member(i, t, member) { ++ const struct btf_type *member_type = btf_type_by_id(btf, ++ member->type); ++ const struct btf_kind_operations *ops; ++ u32 member_offset, bitfield_size; ++ u32 bytes_offset; ++ u8 bits8_offset; ++ ++ if (i) ++ seq_puts(m, seq); ++ ++ member_offset = btf_member_bit_offset(t, member); ++ bitfield_size = btf_member_bitfield_size(t, member); ++ bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); ++ bits8_offset = BITS_PER_BYTE_MASKED(member_offset); ++ if (bitfield_size) { ++ btf_bitfield_seq_show(data + bytes_offset, bits8_offset, ++ bitfield_size, m); ++ } else { ++ ops = btf_type_ops(member_type); ++ ops->seq_show(btf, member_type, member->type, ++ data + bytes_offset, bits8_offset, m); ++ } ++ } ++ seq_puts(m, "}"); ++} ++ ++static struct btf_kind_operations struct_ops = { ++ .check_meta = btf_struct_check_meta, ++ .resolve = btf_struct_resolve, ++ .check_member = btf_struct_check_member, ++ .check_kflag_member = btf_generic_check_kflag_member, ++ .log_details = btf_struct_log, ++ .seq_show = btf_struct_seq_show, ++}; ++ ++static int btf_enum_check_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_bits_off = member->offset; ++ u32 struct_size, bytes_offset; ++ ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member is not byte aligned"); ++ return -EINVAL; ++ } ++ ++ struct_size = struct_type->size; ++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); ++ if (struct_size - bytes_offset < member_type->size) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int btf_enum_check_kflag_member(struct btf_verifier_env *env, ++ const struct btf_type *struct_type, ++ const struct btf_member *member, ++ const struct btf_type *member_type) ++{ ++ u32 struct_bits_off, nr_bits, bytes_end, struct_size; ++ u32 int_bitsize = sizeof(int) * BITS_PER_BYTE; ++ ++ struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); ++ nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); ++ if (!nr_bits) { ++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member is not byte aligned"); ++ return -EINVAL; ++ } ++ ++ nr_bits = int_bitsize; ++ } else if (nr_bits > int_bitsize) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Invalid member bitfield_size"); ++ return -EINVAL; ++ } ++ ++ struct_size = struct_type->size; ++ bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits); ++ if (struct_size < bytes_end) { ++ btf_verifier_log_member(env, struct_type, member, ++ "Member exceeds struct_size"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static s32 btf_enum_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ const struct btf_enum *enums = btf_type_enum(t); ++ struct btf *btf = env->btf; ++ u16 i, nr_enums; ++ u32 meta_needed; ++ ++ nr_enums = btf_type_vlen(t); ++ meta_needed = nr_enums * sizeof(*enums); ++ ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ if (t->size > 8 || !is_power_of_2(t->size)) { ++ btf_verifier_log_type(env, t, "Unexpected size"); ++ return -EINVAL; ++ } ++ ++ /* enum type either no name or a valid one */ ++ if (t->name_off && ++ !btf_name_valid_identifier(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ for (i = 0; i < nr_enums; i++) { ++ if (!btf_name_offset_valid(btf, enums[i].name_off)) { ++ btf_verifier_log(env, "\tInvalid name_offset:%u", ++ enums[i].name_off); ++ return -EINVAL; ++ } ++ ++ /* enum member must have a valid name */ ++ if (!enums[i].name_off || ++ !btf_name_valid_identifier(btf, enums[i].name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ ++ btf_verifier_log(env, "\t%s val=%d\n", ++ __btf_name_by_offset(btf, enums[i].name_off), ++ enums[i].val); ++ } ++ ++ return meta_needed; ++} ++ ++static void btf_enum_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); ++} ++ ++static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t, ++ u32 type_id, void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ const struct btf_enum *enums = btf_type_enum(t); ++ u32 i, nr_enums = btf_type_vlen(t); ++ int v = *(int *)data; ++ ++ for (i = 0; i < nr_enums; i++) { ++ if (v == enums[i].val) { ++ seq_printf(m, "%s", ++ __btf_name_by_offset(btf, ++ enums[i].name_off)); ++ return; ++ } ++ } ++ ++ seq_printf(m, "%d", v); ++} ++ ++static struct btf_kind_operations enum_ops = { ++ .check_meta = btf_enum_check_meta, ++ .resolve = btf_df_resolve, ++ .check_member = btf_enum_check_member, ++ .check_kflag_member = btf_enum_check_kflag_member, ++ .log_details = btf_enum_log, ++ .seq_show = btf_enum_seq_show, ++}; ++ ++static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param); ++ ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ if (t->name_off) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return meta_needed; ++} ++ ++static void btf_func_proto_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ const struct btf_param *args = (const struct btf_param *)(t + 1); ++ u16 nr_args = btf_type_vlen(t), i; ++ ++ btf_verifier_log(env, "return=%u args=(", t->type); ++ if (!nr_args) { ++ btf_verifier_log(env, "void"); ++ goto done; ++ } ++ ++ if (nr_args == 1 && !args[0].type) { ++ /* Only one vararg */ ++ btf_verifier_log(env, "vararg"); ++ goto done; ++ } ++ ++ btf_verifier_log(env, "%u %s", args[0].type, ++ __btf_name_by_offset(env->btf, ++ args[0].name_off)); ++ for (i = 1; i < nr_args - 1; i++) ++ btf_verifier_log(env, ", %u %s", args[i].type, ++ __btf_name_by_offset(env->btf, ++ args[i].name_off)); ++ ++ if (nr_args > 1) { ++ const struct btf_param *last_arg = &args[nr_args - 1]; ++ ++ if (last_arg->type) ++ btf_verifier_log(env, ", %u %s", last_arg->type, ++ __btf_name_by_offset(env->btf, ++ last_arg->name_off)); ++ else ++ btf_verifier_log(env, ", vararg"); ++ } ++ ++done: ++ btf_verifier_log(env, ")"); ++} ++ ++static struct btf_kind_operations func_proto_ops = { ++ .check_meta = btf_func_proto_check_meta, ++ .resolve = btf_df_resolve, ++ /* ++ * BTF_KIND_FUNC_PROTO cannot be directly referred by ++ * a struct's member. ++ * ++ * It should be a funciton pointer instead. ++ * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO) ++ * ++ * Hence, there is no btf_func_check_member(). ++ */ ++ .check_member = btf_df_check_member, ++ .check_kflag_member = btf_df_check_kflag_member, ++ .log_details = btf_func_proto_log, ++ .seq_show = btf_df_seq_show, ++}; ++ ++static s32 btf_func_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ if (!t->name_off || ++ !btf_name_valid_identifier(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return 0; ++} ++ ++static struct btf_kind_operations func_ops = { ++ .check_meta = btf_func_check_meta, ++ .resolve = btf_df_resolve, ++ .check_member = btf_df_check_member, ++ .check_kflag_member = btf_df_check_kflag_member, ++ .log_details = btf_ref_type_log, ++ .seq_show = btf_df_seq_show, ++}; ++ ++static s32 btf_var_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ const struct btf_var *var; ++ u32 meta_needed = sizeof(*var); ++ ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ if (btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen != 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ if (!t->name_off || ++ !__btf_name_valid(env->btf, t->name_off, true)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ /* A var cannot be in type void */ ++ if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { ++ btf_verifier_log_type(env, t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ var = btf_type_var(t); ++ if (var->linkage != BTF_VAR_STATIC && ++ var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { ++ btf_verifier_log_type(env, t, "Linkage not supported"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ return meta_needed; ++} ++ ++static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t) ++{ ++ const struct btf_var *var = btf_type_var(t); ++ ++ btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); ++} ++ ++static const struct btf_kind_operations var_ops = { ++ .check_meta = btf_var_check_meta, ++ .resolve = btf_var_resolve, ++ .check_member = btf_df_check_member, ++ .check_kflag_member = btf_df_check_kflag_member, ++ .log_details = btf_var_log, ++ .seq_show = btf_var_seq_show, ++}; ++ ++static s32 btf_datasec_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ const struct btf_var_secinfo *vsi; ++ u64 last_vsi_end_off = 0, sum = 0; ++ u32 i, meta_needed; ++ ++ meta_needed = btf_type_vlen(t) * sizeof(*vsi); ++ if (meta_left < meta_needed) { ++ btf_verifier_log_basic(env, t, ++ "meta_left:%u meta_needed:%u", ++ meta_left, meta_needed); ++ return -EINVAL; ++ } ++ ++ if (!btf_type_vlen(t)) { ++ btf_verifier_log_type(env, t, "vlen == 0"); ++ return -EINVAL; ++ } ++ ++ if (!t->size) { ++ btf_verifier_log_type(env, t, "size == 0"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_kflag(t)) { ++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); ++ return -EINVAL; ++ } ++ ++ if (!t->name_off || ++ !btf_name_valid_section(env->btf, t->name_off)) { ++ btf_verifier_log_type(env, t, "Invalid name"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_type(env, t, NULL); ++ ++ for_each_vsi(i, t, vsi) { ++ /* A var cannot be in type void */ ++ if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { ++ btf_verifier_log_vsi(env, t, vsi, ++ "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { ++ btf_verifier_log_vsi(env, t, vsi, ++ "Invalid offset"); ++ return -EINVAL; ++ } ++ ++ if (!vsi->size || vsi->size > t->size) { ++ btf_verifier_log_vsi(env, t, vsi, ++ "Invalid size"); ++ return -EINVAL; ++ } ++ ++ last_vsi_end_off = vsi->offset + vsi->size; ++ if (last_vsi_end_off > t->size) { ++ btf_verifier_log_vsi(env, t, vsi, ++ "Invalid offset+size"); ++ return -EINVAL; ++ } ++ ++ btf_verifier_log_vsi(env, t, vsi, NULL); ++ sum += vsi->size; ++ } ++ ++ if (t->size < sum) { ++ btf_verifier_log_type(env, t, "Invalid btf_info size"); ++ return -EINVAL; ++ } ++ ++ return meta_needed; ++} ++ ++static int btf_datasec_resolve(struct btf_verifier_env *env, ++ const struct resolve_vertex *v) ++{ ++ const struct btf_var_secinfo *vsi; ++ struct btf *btf = env->btf; ++ u16 i; ++ ++ for_each_vsi_from(i, v->next_member, v->t, vsi) { ++ u32 var_type_id = vsi->type, type_id, type_size = 0; ++ const struct btf_type *var_type = btf_type_by_id(env->btf, ++ var_type_id); ++ if (!var_type || !btf_type_is_var(var_type)) { ++ btf_verifier_log_vsi(env, v->t, vsi, ++ "Not a VAR kind member"); ++ return -EINVAL; ++ } ++ ++ if (!env_type_is_resolve_sink(env, var_type) && ++ !env_type_is_resolved(env, var_type_id)) { ++ env_stack_set_next_member(env, i + 1); ++ return env_stack_push(env, var_type, var_type_id); ++ } ++ ++ type_id = var_type->type; ++ if (!btf_type_id_size(btf, &type_id, &type_size)) { ++ btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); ++ return -EINVAL; ++ } ++ ++ if (vsi->size < type_size) { ++ btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); ++ return -EINVAL; ++ } ++ } ++ ++ env_stack_pop_resolved(env, 0, 0); ++ return 0; ++} ++ ++static void btf_datasec_log(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); ++} ++ ++static void btf_datasec_seq_show(const struct btf *btf, ++ const struct btf_type *t, u32 type_id, ++ void *data, u8 bits_offset, ++ struct seq_file *m) ++{ ++ const struct btf_var_secinfo *vsi; ++ const struct btf_type *var; ++ u32 i; ++ ++ seq_printf(m, "section (\"%s\") = {", __btf_name_by_offset(btf, t->name_off)); ++ for_each_vsi(i, t, vsi) { ++ var = btf_type_by_id(btf, vsi->type); ++ if (i) ++ seq_puts(m, ","); ++ btf_type_ops(var)->seq_show(btf, var, vsi->type, ++ data + vsi->offset, bits_offset, m); ++ } ++ seq_puts(m, "}"); ++} ++ ++static const struct btf_kind_operations datasec_ops = { ++ .check_meta = btf_datasec_check_meta, ++ .resolve = btf_datasec_resolve, ++ .check_member = btf_df_check_member, ++ .check_kflag_member = btf_df_check_kflag_member, ++ .log_details = btf_datasec_log, ++ .seq_show = btf_datasec_seq_show, ++}; ++ ++static int btf_func_proto_check(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ const struct btf_type *ret_type; ++ const struct btf_param *args; ++ const struct btf *btf; ++ u16 nr_args, i; ++ int err; ++ ++ btf = env->btf; ++ args = (const struct btf_param *)(t + 1); ++ nr_args = btf_type_vlen(t); ++ ++ /* Check func return type which could be "void" (t->type == 0) */ ++ if (t->type) { ++ u32 ret_type_id = t->type; ++ ++ ret_type = btf_type_by_id(btf, ret_type_id); ++ if (!ret_type) { ++ btf_verifier_log_type(env, t, "Invalid return type"); ++ return -EINVAL; ++ } ++ ++ if (btf_type_needs_resolve(ret_type) && ++ !env_type_is_resolved(env, ret_type_id)) { ++ err = btf_resolve(env, ret_type, ret_type_id); ++ if (err) ++ return err; ++ } ++ ++ /* Ensure the return type is a type that has a size */ ++ if (!btf_type_id_size(btf, &ret_type_id, NULL)) { ++ btf_verifier_log_type(env, t, "Invalid return type"); ++ return -EINVAL; ++ } ++ } ++ ++ if (!nr_args) ++ return 0; ++ ++ /* Last func arg type_id could be 0 if it is a vararg */ ++ if (!args[nr_args - 1].type) { ++ if (args[nr_args - 1].name_off) { ++ btf_verifier_log_type(env, t, "Invalid arg#%u", ++ nr_args); ++ return -EINVAL; ++ } ++ nr_args--; ++ } ++ ++ err = 0; ++ for (i = 0; i < nr_args; i++) { ++ const struct btf_type *arg_type; ++ u32 arg_type_id; ++ ++ arg_type_id = args[i].type; ++ arg_type = btf_type_by_id(btf, arg_type_id); ++ if (!arg_type) { ++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); ++ err = -EINVAL; ++ break; ++ } ++ ++ if (args[i].name_off && ++ (!btf_name_offset_valid(btf, args[i].name_off) || ++ !btf_name_valid_identifier(btf, args[i].name_off))) { ++ btf_verifier_log_type(env, t, ++ "Invalid arg#%u", i + 1); ++ err = -EINVAL; ++ break; ++ } ++ ++ if (btf_type_needs_resolve(arg_type) && ++ !env_type_is_resolved(env, arg_type_id)) { ++ err = btf_resolve(env, arg_type, arg_type_id); ++ if (err) ++ break; ++ } ++ ++ if (!btf_type_id_size(btf, &arg_type_id, NULL)) { ++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); ++ err = -EINVAL; ++ break; ++ } ++ } ++ ++ return err; ++} ++ ++static int btf_func_check(struct btf_verifier_env *env, ++ const struct btf_type *t) ++{ ++ const struct btf_type *proto_type; ++ const struct btf_param *args; ++ const struct btf *btf; ++ u16 nr_args, i; ++ ++ btf = env->btf; ++ proto_type = btf_type_by_id(btf, t->type); ++ ++ if (!proto_type || !btf_type_is_func_proto(proto_type)) { ++ btf_verifier_log_type(env, t, "Invalid type_id"); ++ return -EINVAL; ++ } ++ ++ args = (const struct btf_param *)(proto_type + 1); ++ nr_args = btf_type_vlen(proto_type); ++ for (i = 0; i < nr_args; i++) { ++ if (!args[i].name_off && args[i].type) { ++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); ++ return -EINVAL; ++ } ++ } ++ ++ return 0; ++} ++ ++static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { ++ [BTF_KIND_INT] = &int_ops, ++ [BTF_KIND_PTR] = &ptr_ops, ++ [BTF_KIND_ARRAY] = &array_ops, ++ [BTF_KIND_STRUCT] = &struct_ops, ++ [BTF_KIND_UNION] = &struct_ops, ++ [BTF_KIND_ENUM] = &enum_ops, ++ [BTF_KIND_FWD] = &fwd_ops, ++ [BTF_KIND_TYPEDEF] = &modifier_ops, ++ [BTF_KIND_VOLATILE] = &modifier_ops, ++ [BTF_KIND_CONST] = &modifier_ops, ++ [BTF_KIND_RESTRICT] = &modifier_ops, ++ [BTF_KIND_FUNC] = &func_ops, ++ [BTF_KIND_FUNC_PROTO] = &func_proto_ops, ++ [BTF_KIND_VAR] = &var_ops, ++ [BTF_KIND_DATASEC] = &datasec_ops, ++}; ++ ++static s32 btf_check_meta(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 meta_left) ++{ ++ u32 saved_meta_left = meta_left; ++ s32 var_meta_size; ++ ++ if (meta_left < sizeof(*t)) { ++ btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu", ++ env->log_type_id, meta_left, sizeof(*t)); ++ return -EINVAL; ++ } ++ meta_left -= sizeof(*t); ++ ++ if (t->info & ~BTF_INFO_MASK) { ++ btf_verifier_log(env, "[%u] Invalid btf_info:%x", ++ env->log_type_id, t->info); ++ return -EINVAL; ++ } ++ ++ if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || ++ BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { ++ btf_verifier_log(env, "[%u] Invalid kind:%u", ++ env->log_type_id, BTF_INFO_KIND(t->info)); ++ return -EINVAL; ++ } ++ ++ if (!btf_name_offset_valid(env->btf, t->name_off)) { ++ btf_verifier_log(env, "[%u] Invalid name_offset:%u", ++ env->log_type_id, t->name_off); ++ return -EINVAL; ++ } ++ ++ var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); ++ if (var_meta_size < 0) ++ return var_meta_size; ++ ++ meta_left -= var_meta_size; ++ ++ return saved_meta_left - meta_left; ++} ++ ++static int btf_check_all_metas(struct btf_verifier_env *env) ++{ ++ struct btf *btf = env->btf; ++ struct btf_header *hdr; ++ void *cur, *end; ++ ++ hdr = &btf->hdr; ++ cur = btf->nohdr_data + hdr->type_off; ++ end = cur + hdr->type_len; ++ ++ env->log_type_id = 1; ++ while (cur < end) { ++ struct btf_type *t = cur; ++ s32 meta_size; ++ ++ meta_size = btf_check_meta(env, t, end - cur); ++ if (meta_size < 0) ++ return meta_size; ++ ++ btf_add_type(env, t); ++ cur += meta_size; ++ env->log_type_id++; ++ } ++ ++ return 0; ++} ++ ++static bool btf_resolve_valid(struct btf_verifier_env *env, ++ const struct btf_type *t, ++ u32 type_id) ++{ ++ struct btf *btf = env->btf; ++ ++ if (!env_type_is_resolved(env, type_id)) ++ return false; ++ ++ if (btf_type_is_struct(t) || btf_type_is_datasec(t)) ++ return !btf->resolved_ids[type_id] && ++ !btf->resolved_sizes[type_id]; ++ ++ if (btf_type_is_modifier(t) || btf_type_is_ptr(t) || ++ btf_type_is_var(t)) { ++ t = btf_type_id_resolve(btf, &type_id); ++ return t && ++ !btf_type_is_modifier(t) && ++ !btf_type_is_var(t) && ++ !btf_type_is_datasec(t); ++ } ++ ++ if (btf_type_is_array(t)) { ++ const struct btf_array *array = btf_type_array(t); ++ const struct btf_type *elem_type; ++ u32 elem_type_id = array->type; ++ u32 elem_size; ++ ++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); ++ return elem_type && !btf_type_is_modifier(elem_type) && ++ (array->nelems * elem_size == ++ btf->resolved_sizes[type_id]); ++ } ++ ++ return false; ++} ++ ++static int btf_resolve(struct btf_verifier_env *env, ++ const struct btf_type *t, u32 type_id) ++{ ++ u32 save_log_type_id = env->log_type_id; ++ const struct resolve_vertex *v; ++ int err = 0; ++ ++ env->resolve_mode = RESOLVE_TBD; ++ env_stack_push(env, t, type_id); ++ while (!err && (v = env_stack_peak(env))) { ++ env->log_type_id = v->type_id; ++ err = btf_type_ops(v->t)->resolve(env, v); ++ } ++ ++ env->log_type_id = type_id; ++ if (err == -E2BIG) { ++ btf_verifier_log_type(env, t, ++ "Exceeded max resolving depth:%u", ++ MAX_RESOLVE_DEPTH); ++ } else if (err == -EEXIST) { ++ btf_verifier_log_type(env, t, "Loop detected"); ++ } ++ ++ /* Final sanity check */ ++ if (!err && !btf_resolve_valid(env, t, type_id)) { ++ btf_verifier_log_type(env, t, "Invalid resolve state"); ++ err = -EINVAL; ++ } ++ ++ env->log_type_id = save_log_type_id; ++ return err; ++} ++ ++static int btf_check_all_types(struct btf_verifier_env *env) ++{ ++ struct btf *btf = env->btf; ++ u32 type_id; ++ int err; ++ ++ err = env_resolve_init(env); ++ if (err) ++ return err; ++ ++ env->phase++; ++ for (type_id = 1; type_id <= btf->nr_types; type_id++) { ++ const struct btf_type *t = btf_type_by_id(btf, type_id); ++ ++ env->log_type_id = type_id; ++ if (btf_type_needs_resolve(t) && ++ !env_type_is_resolved(env, type_id)) { ++ err = btf_resolve(env, t, type_id); ++ if (err) ++ return err; ++ } ++ ++ if (btf_type_is_func_proto(t)) { ++ err = btf_func_proto_check(env, t); ++ if (err) ++ return err; ++ } ++ ++ if (btf_type_is_func(t)) { ++ err = btf_func_check(env, t); ++ if (err) ++ return err; ++ } ++ } ++ ++ return 0; ++} ++ ++static int btf_parse_type_sec(struct btf_verifier_env *env) ++{ ++ const struct btf_header *hdr = &env->btf->hdr; ++ int err; ++ ++ /* Type section must align to 4 bytes */ ++ if (hdr->type_off & (sizeof(u32) - 1)) { ++ btf_verifier_log(env, "Unaligned type_off"); ++ return -EINVAL; ++ } ++ ++ if (!hdr->type_len) { ++ btf_verifier_log(env, "No type found"); ++ return -EINVAL; ++ } ++ ++ err = btf_check_all_metas(env); ++ if (err) ++ return err; ++ ++ return btf_check_all_types(env); ++} ++ ++static int btf_parse_str_sec(struct btf_verifier_env *env) ++{ ++ const struct btf_header *hdr; ++ struct btf *btf = env->btf; ++ const char *start, *end; ++ ++ hdr = &btf->hdr; ++ start = btf->nohdr_data + hdr->str_off; ++ end = start + hdr->str_len; ++ ++ if (end != btf->data + btf->data_size) { ++ btf_verifier_log(env, "String section is not at the end"); ++ return -EINVAL; ++ } ++ ++ if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || ++ start[0] || end[-1]) { ++ btf_verifier_log(env, "Invalid string section"); ++ return -EINVAL; ++ } ++ ++ btf->strings = start; ++ ++ return 0; ++} ++ ++static const size_t btf_sec_info_offset[] = { ++ offsetof(struct btf_header, type_off), ++ offsetof(struct btf_header, str_off), ++}; ++ ++static int btf_sec_info_cmp(const void *a, const void *b) ++{ ++ const struct btf_sec_info *x = a; ++ const struct btf_sec_info *y = b; ++ ++ return (int)(x->off - y->off) ? : (int)(x->len - y->len); ++} ++ ++static int btf_check_sec_info(struct btf_verifier_env *env, ++ u32 btf_data_size) ++{ ++ struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)]; ++ u32 total, expected_total, i; ++ const struct btf_header *hdr; ++ const struct btf *btf; ++ ++ btf = env->btf; ++ hdr = &btf->hdr; ++ ++ /* Populate the secs from hdr */ ++ for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) ++ secs[i] = *(struct btf_sec_info *)((void *)hdr + ++ btf_sec_info_offset[i]); ++ ++ sort(secs, ARRAY_SIZE(btf_sec_info_offset), ++ sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL); ++ ++ /* Check for gaps and overlap among sections */ ++ total = 0; ++ expected_total = btf_data_size - hdr->hdr_len; ++ for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) { ++ if (expected_total < secs[i].off) { ++ btf_verifier_log(env, "Invalid section offset"); ++ return -EINVAL; ++ } ++ if (total < secs[i].off) { ++ /* gap */ ++ btf_verifier_log(env, "Unsupported section found"); ++ return -EINVAL; ++ } ++ if (total > secs[i].off) { ++ btf_verifier_log(env, "Section overlap found"); ++ return -EINVAL; ++ } ++ if (expected_total - total < secs[i].len) { ++ btf_verifier_log(env, ++ "Total section length too long"); ++ return -EINVAL; ++ } ++ total += secs[i].len; ++ } ++ ++ /* There is data other than hdr and known sections */ ++ if (expected_total != total) { ++ btf_verifier_log(env, "Unsupported section found"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static int btf_parse_hdr(struct btf_verifier_env *env) ++{ ++ u32 hdr_len, hdr_copy, btf_data_size; ++ const struct btf_header *hdr; ++ struct btf *btf; ++ int err; ++ ++ btf = env->btf; ++ btf_data_size = btf->data_size; ++ ++ if (btf_data_size < ++ offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) { ++ btf_verifier_log(env, "hdr_len not found"); ++ return -EINVAL; ++ } ++ ++ hdr = btf->data; ++ hdr_len = hdr->hdr_len; ++ if (btf_data_size < hdr_len) { ++ btf_verifier_log(env, "btf_header not found"); ++ return -EINVAL; ++ } ++ ++ /* Ensure the unsupported header fields are zero */ ++ if (hdr_len > sizeof(btf->hdr)) { ++ u8 *expected_zero = btf->data + sizeof(btf->hdr); ++ u8 *end = btf->data + hdr_len; ++ ++ for (; expected_zero < end; expected_zero++) { ++ if (*expected_zero) { ++ btf_verifier_log(env, "Unsupported btf_header"); ++ return -E2BIG; ++ } ++ } ++ } ++ ++ hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); ++ memcpy(&btf->hdr, btf->data, hdr_copy); ++ ++ hdr = &btf->hdr; ++ ++ btf_verifier_log_hdr(env, btf_data_size); ++ ++ if (hdr->magic != BTF_MAGIC) { ++ btf_verifier_log(env, "Invalid magic"); ++ return -EINVAL; ++ } ++ ++ if (hdr->version != BTF_VERSION) { ++ btf_verifier_log(env, "Unsupported version"); ++ return -ENOTSUPP; ++ } ++ ++ if (hdr->flags) { ++ btf_verifier_log(env, "Unsupported flags"); ++ return -ENOTSUPP; ++ } ++ ++ if (btf_data_size == hdr->hdr_len) { ++ btf_verifier_log(env, "No data"); ++ return -EINVAL; ++ } ++ ++ err = btf_check_sec_info(env, btf_data_size); ++ if (err) ++ return err; ++ ++ return 0; ++} ++ ++static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size, ++ u32 log_level, char __user *log_ubuf, u32 log_size) ++{ ++ struct btf_verifier_env *env = NULL; ++ struct bpf_verifier_log *log; ++ struct btf *btf = NULL; ++ u8 *data; ++ int err; ++ ++ if (btf_data_size > BTF_MAX_SIZE) ++ return ERR_PTR(-E2BIG); ++ ++ env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN); ++ if (!env) ++ return ERR_PTR(-ENOMEM); ++ ++ log = &env->log; ++ if (log_level || log_ubuf || log_size) { ++ /* user requested verbose verifier output ++ * and supplied buffer to store the verification trace ++ */ ++ log->level = log_level; ++ log->ubuf = log_ubuf; ++ log->len_total = log_size; ++ ++ /* log attributes have to be sane */ ++ if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 || ++ !log->level || !log->ubuf) { ++ err = -EINVAL; ++ goto errout; ++ } ++ } ++ ++ btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); ++ if (!btf) { ++ err = -ENOMEM; ++ goto errout; ++ } ++ env->btf = btf; ++ ++ data = kmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN); ++ if (!data) { ++ err = -ENOMEM; ++ goto errout; ++ } ++ ++ btf->data = data; ++ btf->data_size = btf_data_size; ++ ++ if (copy_from_user(data, btf_data, btf_data_size)) { ++ err = -EFAULT; ++ goto errout; ++ } ++ ++ err = btf_parse_hdr(env); ++ if (err) ++ goto errout; ++ ++ btf->nohdr_data = btf->data + btf->hdr.hdr_len; ++ ++ err = btf_parse_str_sec(env); ++ if (err) ++ goto errout; ++ ++ err = btf_parse_type_sec(env); ++ if (err) ++ goto errout; ++ ++ if (log->level && bpf_verifier_log_full(log)) { ++ err = -ENOSPC; ++ goto errout; ++ } ++ ++ btf_verifier_env_free(env); ++ refcount_set(&btf->refcnt, 1); ++ return btf; ++ ++errout: ++ btf_verifier_env_free(env); ++ if (btf) ++ btf_free(btf); ++ return ERR_PTR(err); ++} ++ ++void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, ++ struct seq_file *m) ++{ ++ const struct btf_type *t = btf_type_by_id(btf, type_id); ++ ++ btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m); ++} ++ ++#ifdef CONFIG_PROC_FS ++static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) ++{ ++ const struct btf *btf = filp->private_data; ++ ++ seq_printf(m, "btf_id:\t%u\n", btf->id); ++} ++#endif ++ ++static int btf_release(struct inode *inode, struct file *filp) ++{ ++ btf_put(filp->private_data); ++ return 0; ++} ++ ++const struct file_operations btf_fops = { ++#ifdef CONFIG_PROC_FS ++ .show_fdinfo = bpf_btf_show_fdinfo, ++#endif ++ .release = btf_release, ++}; ++ ++static int __btf_new_fd(struct btf *btf) ++{ ++ return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); ++} ++ ++int btf_new_fd(const union bpf_attr *attr) ++{ ++ struct btf *btf; ++ int ret; ++ ++ btf = btf_parse(u64_to_user_ptr(attr->btf), ++ attr->btf_size, attr->btf_log_level, ++ u64_to_user_ptr(attr->btf_log_buf), ++ attr->btf_log_size); ++ if (IS_ERR(btf)) ++ return PTR_ERR(btf); ++ ++ ret = btf_alloc_id(btf); ++ if (ret) { ++ btf_free(btf); ++ return ret; ++ } ++ ++ /* ++ * The BTF ID is published to the userspace. ++ * All BTF free must go through call_rcu() from ++ * now on (i.e. free by calling btf_put()). ++ */ ++ ++ ret = __btf_new_fd(btf); ++ if (ret < 0) ++ btf_put(btf); ++ ++ return ret; ++} ++ ++struct btf *btf_get_by_fd(int fd) ++{ ++ struct btf *btf; ++ struct fd f; ++ ++ f = fdget(fd); ++ ++ if (!f.file) ++ return ERR_PTR(-EBADF); ++ ++ if (f.file->f_op != &btf_fops) { ++ fdput(f); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ btf = f.file->private_data; ++ refcount_inc(&btf->refcnt); ++ fdput(f); ++ ++ return btf; ++} ++ ++int btf_get_info_by_fd(const struct btf *btf, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct bpf_btf_info __user *uinfo; ++ struct bpf_btf_info info; ++ u32 info_copy, btf_copy; ++ void __user *ubtf; ++ u32 uinfo_len; ++ ++ uinfo = u64_to_user_ptr(attr->info.info); ++ uinfo_len = attr->info.info_len; ++ ++ info_copy = min_t(u32, uinfo_len, sizeof(info)); ++ memset(&info, 0, sizeof(info)); ++ if (copy_from_user(&info, uinfo, info_copy)) ++ return -EFAULT; ++ ++ info.id = btf->id; ++ ubtf = u64_to_user_ptr(info.btf); ++ btf_copy = min_t(u32, btf->data_size, info.btf_size); ++ if (copy_to_user(ubtf, btf->data, btf_copy)) ++ return -EFAULT; ++ info.btf_size = btf->data_size; ++ ++ if (copy_to_user(uinfo, &info, info_copy) || ++ put_user(info_copy, &uattr->info.info_len)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++int btf_get_fd_by_id(u32 id) ++{ ++ struct btf *btf; ++ int fd; ++ ++ rcu_read_lock(); ++ btf = idr_find(&btf_idr, id); ++ if (!btf || !refcount_inc_not_zero(&btf->refcnt)) ++ btf = ERR_PTR(-ENOENT); ++ rcu_read_unlock(); ++ ++ if (IS_ERR(btf)) ++ return PTR_ERR(btf); ++ ++ fd = __btf_new_fd(btf); ++ if (fd < 0) ++ btf_put(btf); ++ ++ return fd; ++} ++ ++u32 btf_id(const struct btf *btf) ++{ ++ return btf->id; ++} +--- /dev/null ++++ b/kernel/bpf/cgroup.c +@@ -0,0 +1,1581 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Functions to manage eBPF programs attached to cgroups ++ * ++ * Copyright (c) 2016 Daniel Mack ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "../cgroup/cgroup-internal.h" ++ ++DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key); ++EXPORT_SYMBOL(cgroup_bpf_enabled_key); ++ ++void cgroup_bpf_offline(struct cgroup *cgrp) ++{ ++ cgroup_get(cgrp); ++ percpu_ref_kill(&cgrp->bpf.refcnt); ++} ++ ++/** ++ * cgroup_bpf_release() - put references of all bpf programs and ++ * release all cgroup bpf data ++ * @work: work structure embedded into the cgroup to modify ++ */ ++static void cgroup_bpf_release(struct work_struct *work) ++{ ++ struct cgroup *p, *cgrp = container_of(work, struct cgroup, ++ bpf.release_work); ++ enum bpf_cgroup_storage_type stype; ++ struct bpf_prog_array *old_array; ++ unsigned int type; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) { ++ struct list_head *progs = &cgrp->bpf.progs[type]; ++ struct bpf_prog_list *pl, *tmp; ++ ++ list_for_each_entry_safe(pl, tmp, progs, node) { ++ list_del(&pl->node); ++ bpf_prog_put(pl->prog); ++ for_each_cgroup_storage_type(stype) { ++ bpf_cgroup_storage_unlink(pl->storage[stype]); ++ bpf_cgroup_storage_free(pl->storage[stype]); ++ } ++ kfree(pl); ++ static_branch_dec(&cgroup_bpf_enabled_key); ++ } ++ old_array = rcu_dereference_protected( ++ cgrp->bpf.effective[type], ++ lockdep_is_held(&cgroup_mutex)); ++ bpf_prog_array_free(old_array); ++ } ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) ++ cgroup_bpf_put(p); ++ ++ percpu_ref_exit(&cgrp->bpf.refcnt); ++ cgroup_put(cgrp); ++} ++ ++/** ++ * cgroup_bpf_release_fn() - callback used to schedule releasing ++ * of bpf cgroup data ++ * @ref: percpu ref counter structure ++ */ ++static void cgroup_bpf_release_fn(struct percpu_ref *ref) ++{ ++ struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); ++ ++ INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release); ++ queue_work(system_wq, &cgrp->bpf.release_work); ++} ++ ++/* count number of elements in the list. ++ * it's slow but the list cannot be long ++ */ ++static u32 prog_list_length(struct list_head *head) ++{ ++ struct bpf_prog_list *pl; ++ u32 cnt = 0; ++ ++ list_for_each_entry(pl, head, node) { ++ if (!pl->prog) ++ continue; ++ cnt++; ++ } ++ return cnt; ++} ++ ++/* if parent has non-overridable prog attached, ++ * disallow attaching new programs to the descendent cgroup. ++ * if parent has overridable or multi-prog, allow attaching ++ */ ++static bool hierarchy_allows_attach(struct cgroup *cgrp, ++ enum bpf_attach_type type, ++ u32 new_flags) ++{ ++ struct cgroup *p; ++ ++ p = cgroup_parent(cgrp); ++ if (!p) ++ return true; ++ do { ++ u32 flags = p->bpf.flags[type]; ++ u32 cnt; ++ ++ if (flags & BPF_F_ALLOW_MULTI) ++ return true; ++ cnt = prog_list_length(&p->bpf.progs[type]); ++ WARN_ON_ONCE(cnt > 1); ++ if (cnt == 1) ++ return !!(flags & BPF_F_ALLOW_OVERRIDE); ++ p = cgroup_parent(p); ++ } while (p); ++ return true; ++} ++ ++/* compute a chain of effective programs for a given cgroup: ++ * start from the list of programs in this cgroup and add ++ * all parent programs. ++ * Note that parent's F_ALLOW_OVERRIDE-type program is yielding ++ * to programs in this cgroup ++ */ ++static int compute_effective_progs(struct cgroup *cgrp, ++ enum bpf_attach_type type, ++ struct bpf_prog_array **array) ++{ ++ enum bpf_cgroup_storage_type stype; ++ struct bpf_prog_array *progs; ++ struct bpf_prog_list *pl; ++ struct cgroup *p = cgrp; ++ int cnt = 0; ++ ++ /* count number of effective programs by walking parents */ ++ do { ++ if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) ++ cnt += prog_list_length(&p->bpf.progs[type]); ++ p = cgroup_parent(p); ++ } while (p); ++ ++ progs = bpf_prog_array_alloc(cnt, GFP_KERNEL); ++ if (!progs) ++ return -ENOMEM; ++ ++ /* populate the array with effective progs */ ++ cnt = 0; ++ p = cgrp; ++ do { ++ if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) ++ continue; ++ ++ list_for_each_entry(pl, &p->bpf.progs[type], node) { ++ if (!pl->prog) ++ continue; ++ ++ progs->items[cnt].prog = pl->prog; ++ for_each_cgroup_storage_type(stype) ++ progs->items[cnt].cgroup_storage[stype] = ++ pl->storage[stype]; ++ cnt++; ++ } ++ } while ((p = cgroup_parent(p))); ++ ++ *array = progs; ++ return 0; ++} ++ ++static void activate_effective_progs(struct cgroup *cgrp, ++ enum bpf_attach_type type, ++ struct bpf_prog_array *old_array) ++{ ++ rcu_swap_protected(cgrp->bpf.effective[type], old_array, ++ lockdep_is_held(&cgroup_mutex)); ++ /* free prog array after grace period, since __cgroup_bpf_run_*() ++ * might be still walking the array ++ */ ++ bpf_prog_array_free(old_array); ++} ++ ++/** ++ * cgroup_bpf_inherit() - inherit effective programs from parent ++ * @cgrp: the cgroup to modify ++ */ ++int cgroup_bpf_inherit(struct cgroup *cgrp) ++{ ++/* has to use marco instead of const int, since compiler thinks ++ * that array below is variable length ++ */ ++#define NR ARRAY_SIZE(cgrp->bpf.effective) ++ struct bpf_prog_array *arrays[NR] = {}; ++ struct cgroup *p; ++ int ret, i; ++ ++ ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, ++ GFP_KERNEL); ++ if (ret) ++ return ret; ++ ++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) ++ cgroup_bpf_get(p); ++ ++ for (i = 0; i < NR; i++) ++ INIT_LIST_HEAD(&cgrp->bpf.progs[i]); ++ ++ for (i = 0; i < NR; i++) ++ if (compute_effective_progs(cgrp, i, &arrays[i])) ++ goto cleanup; ++ ++ for (i = 0; i < NR; i++) ++ activate_effective_progs(cgrp, i, arrays[i]); ++ ++ return 0; ++cleanup: ++ for (i = 0; i < NR; i++) ++ bpf_prog_array_free(arrays[i]); ++ ++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) ++ cgroup_bpf_put(p); ++ ++ percpu_ref_exit(&cgrp->bpf.refcnt); ++ ++ return -ENOMEM; ++} ++ ++static int update_effective_progs(struct cgroup *cgrp, ++ enum bpf_attach_type type) ++{ ++ struct cgroup_subsys_state *css; ++ int err; ++ ++ /* allocate and recompute effective prog arrays */ ++ css_for_each_descendant_pre(css, &cgrp->self) { ++ struct cgroup *desc = container_of(css, struct cgroup, self); ++ ++ if (percpu_ref_is_zero(&desc->bpf.refcnt)) ++ continue; ++ ++ err = compute_effective_progs(desc, type, &desc->bpf.inactive); ++ if (err) ++ goto cleanup; ++ } ++ ++ /* all allocations were successful. Activate all prog arrays */ ++ css_for_each_descendant_pre(css, &cgrp->self) { ++ struct cgroup *desc = container_of(css, struct cgroup, self); ++ ++ if (percpu_ref_is_zero(&desc->bpf.refcnt)) { ++ if (unlikely(desc->bpf.inactive)) { ++ bpf_prog_array_free(desc->bpf.inactive); ++ desc->bpf.inactive = NULL; ++ } ++ continue; ++ } ++ ++ activate_effective_progs(desc, type, desc->bpf.inactive); ++ desc->bpf.inactive = NULL; ++ } ++ ++ return 0; ++ ++cleanup: ++ /* oom while computing effective. Free all computed effective arrays ++ * since they were not activated ++ */ ++ css_for_each_descendant_pre(css, &cgrp->self) { ++ struct cgroup *desc = container_of(css, struct cgroup, self); ++ ++ bpf_prog_array_free(desc->bpf.inactive); ++ desc->bpf.inactive = NULL; ++ } ++ ++ return err; ++} ++ ++#define BPF_CGROUP_MAX_PROGS 64 ++ ++/** ++ * __cgroup_bpf_attach() - Attach the program to a cgroup, and ++ * propagate the change to descendants ++ * @cgrp: The cgroup which descendants to traverse ++ * @prog: A program to attach ++ * @type: Type of attach operation ++ * @flags: Option flags ++ * ++ * Must be called with cgroup_mutex held. ++ */ ++int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type, u32 flags) ++{ ++ struct list_head *progs = &cgrp->bpf.progs[type]; ++ struct bpf_prog *old_prog = NULL; ++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; ++ struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; ++ enum bpf_cgroup_storage_type stype; ++ struct bpf_prog_list *pl; ++ bool pl_was_allocated; ++ int err; ++ ++ if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) ++ /* invalid combination */ ++ return -EINVAL; ++ ++ if (!hierarchy_allows_attach(cgrp, type, flags)) ++ return -EPERM; ++ ++ if (!list_empty(progs) && cgrp->bpf.flags[type] != flags) ++ /* Disallow attaching non-overridable on top ++ * of existing overridable in this cgroup. ++ * Disallow attaching multi-prog if overridable or none ++ */ ++ return -EPERM; ++ ++ if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) ++ return -E2BIG; ++ ++ for_each_cgroup_storage_type(stype) { ++ storage[stype] = bpf_cgroup_storage_alloc(prog, stype); ++ if (IS_ERR(storage[stype])) { ++ storage[stype] = NULL; ++ for_each_cgroup_storage_type(stype) ++ bpf_cgroup_storage_free(storage[stype]); ++ return -ENOMEM; ++ } ++ } ++ ++ if (flags & BPF_F_ALLOW_MULTI) { ++ list_for_each_entry(pl, progs, node) { ++ if (pl->prog == prog) { ++ /* disallow attaching the same prog twice */ ++ for_each_cgroup_storage_type(stype) ++ bpf_cgroup_storage_free(storage[stype]); ++ return -EINVAL; ++ } ++ } ++ ++ pl = kmalloc(sizeof(*pl), GFP_KERNEL); ++ if (!pl) { ++ for_each_cgroup_storage_type(stype) ++ bpf_cgroup_storage_free(storage[stype]); ++ return -ENOMEM; ++ } ++ ++ pl_was_allocated = true; ++ pl->prog = prog; ++ for_each_cgroup_storage_type(stype) ++ pl->storage[stype] = storage[stype]; ++ list_add_tail(&pl->node, progs); ++ } else { ++ if (list_empty(progs)) { ++ pl = kmalloc(sizeof(*pl), GFP_KERNEL); ++ if (!pl) { ++ for_each_cgroup_storage_type(stype) ++ bpf_cgroup_storage_free(storage[stype]); ++ return -ENOMEM; ++ } ++ pl_was_allocated = true; ++ list_add_tail(&pl->node, progs); ++ } else { ++ pl = list_first_entry(progs, typeof(*pl), node); ++ old_prog = pl->prog; ++ for_each_cgroup_storage_type(stype) { ++ old_storage[stype] = pl->storage[stype]; ++ bpf_cgroup_storage_unlink(old_storage[stype]); ++ } ++ pl_was_allocated = false; ++ } ++ pl->prog = prog; ++ for_each_cgroup_storage_type(stype) ++ pl->storage[stype] = storage[stype]; ++ } ++ ++ cgrp->bpf.flags[type] = flags; ++ ++ err = update_effective_progs(cgrp, type); ++ if (err) ++ goto cleanup; ++ ++ static_branch_inc(&cgroup_bpf_enabled_key); ++ for_each_cgroup_storage_type(stype) { ++ if (!old_storage[stype]) ++ continue; ++ bpf_cgroup_storage_free(old_storage[stype]); ++ } ++ if (old_prog) { ++ bpf_prog_put(old_prog); ++ static_branch_dec(&cgroup_bpf_enabled_key); ++ } ++ for_each_cgroup_storage_type(stype) ++ bpf_cgroup_storage_link(storage[stype], cgrp, type); ++ return 0; ++ ++cleanup: ++ /* and cleanup the prog list */ ++ pl->prog = old_prog; ++ for_each_cgroup_storage_type(stype) { ++ bpf_cgroup_storage_free(pl->storage[stype]); ++ pl->storage[stype] = old_storage[stype]; ++ bpf_cgroup_storage_link(old_storage[stype], cgrp, type); ++ } ++ if (pl_was_allocated) { ++ list_del(&pl->node); ++ kfree(pl); ++ } ++ return err; ++} ++ ++/** ++ * __cgroup_bpf_detach() - Detach the program from a cgroup, and ++ * propagate the change to descendants ++ * @cgrp: The cgroup which descendants to traverse ++ * @prog: A program to detach or NULL ++ * @type: Type of detach operation ++ * ++ * Must be called with cgroup_mutex held. ++ */ ++int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, ++ enum bpf_attach_type type) ++{ ++ struct list_head *progs = &cgrp->bpf.progs[type]; ++ enum bpf_cgroup_storage_type stype; ++ u32 flags = cgrp->bpf.flags[type]; ++ struct bpf_prog *old_prog = NULL; ++ struct bpf_prog_list *pl; ++ int err; ++ ++ if (flags & BPF_F_ALLOW_MULTI) { ++ if (!prog) ++ /* to detach MULTI prog the user has to specify valid FD ++ * of the program to be detached ++ */ ++ return -EINVAL; ++ } else { ++ if (list_empty(progs)) ++ /* report error when trying to detach and nothing is attached */ ++ return -ENOENT; ++ } ++ ++ if (flags & BPF_F_ALLOW_MULTI) { ++ /* find the prog and detach it */ ++ list_for_each_entry(pl, progs, node) { ++ if (pl->prog != prog) ++ continue; ++ old_prog = prog; ++ /* mark it deleted, so it's ignored while ++ * recomputing effective ++ */ ++ pl->prog = NULL; ++ break; ++ } ++ if (!old_prog) ++ return -ENOENT; ++ } else { ++ /* to maintain backward compatibility NONE and OVERRIDE cgroups ++ * allow detaching with invalid FD (prog==NULL) ++ */ ++ pl = list_first_entry(progs, typeof(*pl), node); ++ old_prog = pl->prog; ++ pl->prog = NULL; ++ } ++ ++ err = update_effective_progs(cgrp, type); ++ if (err) ++ goto cleanup; ++ ++ /* now can actually delete it from this cgroup list */ ++ list_del(&pl->node); ++ for_each_cgroup_storage_type(stype) { ++ bpf_cgroup_storage_unlink(pl->storage[stype]); ++ bpf_cgroup_storage_free(pl->storage[stype]); ++ } ++ kfree(pl); ++ if (list_empty(progs)) ++ /* last program was detached, reset flags to zero */ ++ cgrp->bpf.flags[type] = 0; ++ ++ bpf_prog_put(old_prog); ++ static_branch_dec(&cgroup_bpf_enabled_key); ++ return 0; ++ ++cleanup: ++ /* and restore back old_prog */ ++ pl->prog = old_prog; ++ return err; ++} ++ ++/* Must be called with cgroup_mutex held to avoid races. */ ++int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); ++ enum bpf_attach_type type = attr->query.attach_type; ++ struct list_head *progs = &cgrp->bpf.progs[type]; ++ u32 flags = cgrp->bpf.flags[type]; ++ struct bpf_prog_array *effective; ++ int cnt, ret = 0, i; ++ ++ effective = rcu_dereference_protected(cgrp->bpf.effective[type], ++ lockdep_is_held(&cgroup_mutex)); ++ ++ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) ++ cnt = bpf_prog_array_length(effective); ++ else ++ cnt = prog_list_length(progs); ++ ++ if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) ++ return -EFAULT; ++ if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt))) ++ return -EFAULT; ++ if (attr->query.prog_cnt == 0 || !prog_ids || !cnt) ++ /* return early if user requested only program count + flags */ ++ return 0; ++ if (attr->query.prog_cnt < cnt) { ++ cnt = attr->query.prog_cnt; ++ ret = -ENOSPC; ++ } ++ ++ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) { ++ return bpf_prog_array_copy_to_user(effective, prog_ids, cnt); ++ } else { ++ struct bpf_prog_list *pl; ++ u32 id; ++ ++ i = 0; ++ list_for_each_entry(pl, progs, node) { ++ id = pl->prog->aux->id; ++ if (copy_to_user(prog_ids + i, &id, sizeof(id))) ++ return -EFAULT; ++ if (++i == cnt) ++ break; ++ } ++ } ++ return ret; ++} ++ ++int cgroup_bpf_prog_attach(const union bpf_attr *attr, ++ enum bpf_prog_type ptype, struct bpf_prog *prog) ++{ ++ struct cgroup *cgrp; ++ int ret; ++ ++ cgrp = cgroup_get_from_fd(attr->target_fd); ++ if (IS_ERR(cgrp)) ++ return PTR_ERR(cgrp); ++ ++ ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type, ++ attr->attach_flags); ++ cgroup_put(cgrp); ++ return ret; ++} ++ ++int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype) ++{ ++ struct bpf_prog *prog; ++ struct cgroup *cgrp; ++ int ret; ++ ++ cgrp = cgroup_get_from_fd(attr->target_fd); ++ if (IS_ERR(cgrp)) ++ return PTR_ERR(cgrp); ++ ++ prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); ++ if (IS_ERR(prog)) ++ prog = NULL; ++ ++ ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0); ++ if (prog) ++ bpf_prog_put(prog); ++ ++ cgroup_put(cgrp); ++ return ret; ++} ++ ++int cgroup_bpf_prog_query(const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct cgroup *cgrp; ++ int ret; ++ ++ cgrp = cgroup_get_from_fd(attr->query.target_fd); ++ if (IS_ERR(cgrp)) ++ return PTR_ERR(cgrp); ++ ++ ret = cgroup_bpf_query(cgrp, attr, uattr); ++ ++ cgroup_put(cgrp); ++ return ret; ++} ++ ++/** ++ * __cgroup_bpf_run_filter_skb() - Run a program for packet filtering ++ * @sk: The socket sending or receiving traffic ++ * @skb: The skb that is being sent or received ++ * @type: The type of program to be exectuted ++ * ++ * If no socket is passed, or the socket is not of type INET or INET6, ++ * this function does nothing and returns 0. ++ * ++ * The program type passed in via @type must be suitable for network ++ * filtering. No further check is performed to assert that. ++ * ++ * For egress packets, this function can return: ++ * NET_XMIT_SUCCESS (0) - continue with packet output ++ * NET_XMIT_DROP (1) - drop packet and notify TCP to call cwr ++ * NET_XMIT_CN (2) - continue with packet output and notify TCP ++ * to call cwr ++ * -EPERM - drop packet ++ * ++ * For ingress packets, this function will return -EPERM if any ++ * attached program was found and if it returned != 1 during execution. ++ * Otherwise 0 is returned. ++ */ ++int __cgroup_bpf_run_filter_skb(struct sock *sk, ++ struct sk_buff *skb, ++ enum bpf_attach_type type) ++{ ++ unsigned int offset = skb->data - skb_network_header(skb); ++ struct sock *save_sk; ++ void *saved_data_end; ++ struct cgroup *cgrp; ++ int ret; ++ ++ if (!sk || !sk_fullsock(sk)) ++ return 0; ++ ++ if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6) ++ return 0; ++ ++ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ save_sk = skb->sk; ++ skb->sk = sk; ++ __skb_push(skb, offset); ++ ++ /* compute pointers for the bpf prog */ ++ bpf_compute_and_save_data_end(skb, &saved_data_end); ++ ++ if (type == BPF_CGROUP_INET_EGRESS) { ++ ret = BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY( ++ cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb); ++ } else { ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb, ++ __bpf_prog_run_save_cb); ++ ret = (ret == 1 ? 0 : -EPERM); ++ } ++ bpf_restore_data_end(skb, saved_data_end); ++ __skb_pull(skb, offset); ++ skb->sk = save_sk; ++ ++ return ret; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb); ++ ++/** ++ * __cgroup_bpf_run_filter_sk() - Run a program on a sock ++ * @sk: sock structure to manipulate ++ * @type: The type of program to be exectuted ++ * ++ * socket is passed is expected to be of type INET or INET6. ++ * ++ * The program type passed in via @type must be suitable for sock ++ * filtering. No further check is performed to assert that. ++ * ++ * This function will return %-EPERM if any if an attached program was found ++ * and if it returned != 1 during execution. In all other cases, 0 is returned. ++ */ ++int __cgroup_bpf_run_filter_sk(struct sock *sk, ++ enum bpf_attach_type type) ++{ ++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ int ret; ++ ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN); ++ return ret == 1 ? 0 : -EPERM; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); ++ ++/** ++ * __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and ++ * provided by user sockaddr ++ * @sk: sock struct that will use sockaddr ++ * @uaddr: sockaddr struct provided by user ++ * @type: The type of program to be exectuted ++ * @t_ctx: Pointer to attach type specific context ++ * ++ * socket is expected to be of type INET or INET6. ++ * ++ * This function will return %-EPERM if an attached program is found and ++ * returned value != 1 during execution. In all other cases, 0 is returned. ++ */ ++int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, ++ struct sockaddr *uaddr, ++ enum bpf_attach_type type, ++ void *t_ctx) ++{ ++ struct bpf_sock_addr_kern ctx = { ++ .sk = sk, ++ .uaddr = uaddr, ++ .t_ctx = t_ctx, ++ }; ++ struct sockaddr_storage unspec; ++ struct cgroup *cgrp; ++ int ret; ++ ++ /* Check socket family since not all sockets represent network ++ * endpoint (e.g. AF_UNIX). ++ */ ++ if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6) ++ return 0; ++ ++ if (!ctx.uaddr) { ++ memset(&unspec, 0, sizeof(unspec)); ++ ctx.uaddr = (struct sockaddr *)&unspec; ++ } ++ ++ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN); ++ ++ return ret == 1 ? 0 : -EPERM; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr); ++ ++/** ++ * __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock ++ * @sk: socket to get cgroup from ++ * @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains ++ * sk with connection information (IP addresses, etc.) May not contain ++ * cgroup info if it is a req sock. ++ * @type: The type of program to be exectuted ++ * ++ * socket passed is expected to be of type INET or INET6. ++ * ++ * The program type passed in via @type must be suitable for sock_ops ++ * filtering. No further check is performed to assert that. ++ * ++ * This function will return %-EPERM if any if an attached program was found ++ * and if it returned != 1 during execution. In all other cases, 0 is returned. ++ */ ++int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, ++ struct bpf_sock_ops_kern *sock_ops, ++ enum bpf_attach_type type) ++{ ++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ int ret; ++ ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops, ++ BPF_PROG_RUN); ++ return ret == 1 ? 0 : -EPERM; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops); ++ ++int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, ++ short access, enum bpf_attach_type type) ++{ ++ struct cgroup *cgrp; ++ struct bpf_cgroup_dev_ctx ctx = { ++ .access_type = (access << 16) | dev_type, ++ .major = major, ++ .minor = minor, ++ }; ++ int allow = 1; ++ ++ rcu_read_lock(); ++ cgrp = task_dfl_cgroup(current); ++ allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, ++ BPF_PROG_RUN); ++ rcu_read_unlock(); ++ ++ return !allow; ++} ++EXPORT_SYMBOL(__cgroup_bpf_check_dev_permission); ++ ++static const struct bpf_func_proto * ++cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ case BPF_FUNC_map_lookup_elem: ++ return &bpf_map_lookup_elem_proto; ++ case BPF_FUNC_map_update_elem: ++ return &bpf_map_update_elem_proto; ++ case BPF_FUNC_map_delete_elem: ++ return &bpf_map_delete_elem_proto; ++ case BPF_FUNC_map_push_elem: ++ return &bpf_map_push_elem_proto; ++ case BPF_FUNC_map_pop_elem: ++ return &bpf_map_pop_elem_proto; ++ case BPF_FUNC_map_peek_elem: ++ return &bpf_map_peek_elem_proto; ++ case BPF_FUNC_get_current_uid_gid: ++ return &bpf_get_current_uid_gid_proto; ++ case BPF_FUNC_get_local_storage: ++ return &bpf_get_local_storage_proto; ++ case BPF_FUNC_get_current_cgroup_id: ++ return &bpf_get_current_cgroup_id_proto; ++ case BPF_FUNC_trace_printk: ++ if (capable(CAP_SYS_ADMIN)) ++ return bpf_get_trace_printk_proto(); ++ /* fall through */ ++ default: ++ return NULL; ++ } ++} ++ ++static const struct bpf_func_proto * ++cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ return cgroup_base_func_proto(func_id, prog); ++} ++ ++static bool cgroup_dev_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ const int size_default = sizeof(__u32); ++ ++ if (type == BPF_WRITE) ++ return false; ++ ++ if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx)) ++ return false; ++ /* The verifier guarantees that size > 0. */ ++ if (off % size != 0) ++ return false; ++ ++ switch (off) { ++ case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type): ++ bpf_ctx_record_field_size(info, size_default); ++ if (!bpf_ctx_narrow_access_ok(off, size, size_default)) ++ return false; ++ break; ++ default: ++ if (size != size_default) ++ return false; ++ } ++ ++ return true; ++} ++ ++const struct bpf_prog_ops cg_dev_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops cg_dev_verifier_ops = { ++ .get_func_proto = cgroup_dev_func_proto, ++ .is_valid_access = cgroup_dev_is_valid_access, ++}; ++ ++/** ++ * __cgroup_bpf_run_filter_sysctl - Run a program on sysctl ++ * ++ * @head: sysctl table header ++ * @table: sysctl table ++ * @write: sysctl is being read (= 0) or written (= 1) ++ * @buf: pointer to buffer passed by user space ++ * @pcount: value-result argument: value is size of buffer pointed to by @buf, ++ * result is size of @new_buf if program set new value, initial value ++ * otherwise ++ * @ppos: value-result argument: value is position at which read from or write ++ * to sysctl is happening, result is new position if program overrode it, ++ * initial value otherwise ++ * @new_buf: pointer to pointer to new buffer that will be allocated if program ++ * overrides new value provided by user space on sysctl write ++ * NOTE: it's caller responsibility to free *new_buf if it was set ++ * @type: type of program to be executed ++ * ++ * Program is run when sysctl is being accessed, either read or written, and ++ * can allow or deny such access. ++ * ++ * This function will return %-EPERM if an attached program is found and ++ * returned value != 1 during execution. In all other cases 0 is returned. ++ */ ++int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, ++ struct ctl_table *table, int write, ++ void __user *buf, size_t *pcount, ++ loff_t *ppos, void **new_buf, ++ enum bpf_attach_type type) ++{ ++ struct bpf_sysctl_kern ctx = { ++ .head = head, ++ .table = table, ++ .write = write, ++ .ppos = ppos, ++ .cur_val = NULL, ++ .cur_len = PAGE_SIZE, ++ .new_val = NULL, ++ .new_len = 0, ++ .new_updated = 0, ++ }; ++ struct cgroup *cgrp; ++ int ret; ++ ++ ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL); ++ if (ctx.cur_val) { ++ mm_segment_t old_fs; ++ loff_t pos = 0; ++ ++ old_fs = get_fs(); ++ set_fs(KERNEL_DS); ++ if (table->proc_handler(table, 0, (void __user *)ctx.cur_val, ++ &ctx.cur_len, &pos)) { ++ /* Let BPF program decide how to proceed. */ ++ ctx.cur_len = 0; ++ } ++ set_fs(old_fs); ++ } else { ++ /* Let BPF program decide how to proceed. */ ++ ctx.cur_len = 0; ++ } ++ ++ if (write && buf && *pcount) { ++ /* BPF program should be able to override new value with a ++ * buffer bigger than provided by user. ++ */ ++ ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL); ++ ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount); ++ if (!ctx.new_val || ++ copy_from_user(ctx.new_val, buf, ctx.new_len)) ++ /* Let BPF program decide how to proceed. */ ++ ctx.new_len = 0; ++ } ++ ++ rcu_read_lock(); ++ cgrp = task_dfl_cgroup(current); ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN); ++ rcu_read_unlock(); ++ ++ kfree(ctx.cur_val); ++ ++ if (ret == 1 && ctx.new_updated) { ++ *new_buf = ctx.new_val; ++ *pcount = ctx.new_len; ++ } else { ++ kfree(ctx.new_val); ++ } ++ ++ return ret == 1 ? 0 : -EPERM; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl); ++ ++#ifdef CONFIG_NET ++static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, ++ enum bpf_attach_type attach_type) ++{ ++ struct bpf_prog_array *prog_array; ++ bool empty; ++ ++ rcu_read_lock(); ++ prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]); ++ empty = bpf_prog_array_is_empty(prog_array); ++ rcu_read_unlock(); ++ ++ return empty; ++} ++ ++static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) ++{ ++ if (unlikely(max_optlen < 0)) ++ return -EINVAL; ++ ++ if (unlikely(max_optlen > PAGE_SIZE)) { ++ /* We don't expose optvals that are greater than PAGE_SIZE ++ * to the BPF program. ++ */ ++ max_optlen = PAGE_SIZE; ++ } ++ ++ ctx->optval = kzalloc(max_optlen, GFP_USER); ++ if (!ctx->optval) ++ return -ENOMEM; ++ ++ ctx->optval_end = ctx->optval + max_optlen; ++ ++ return max_optlen; ++} ++ ++static void sockopt_free_buf(struct bpf_sockopt_kern *ctx) ++{ ++ kfree(ctx->optval); ++} ++ ++int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, ++ int *optname, char __user *optval, ++ int *optlen, char **kernel_optval) ++{ ++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ struct bpf_sockopt_kern ctx = { ++ .sk = sk, ++ .level = *level, ++ .optname = *optname, ++ }; ++ int ret, max_optlen; ++ ++ /* Opportunistic check to see whether we have any BPF program ++ * attached to the hook so we don't waste time allocating ++ * memory and locking the socket. ++ */ ++ if (!cgroup_bpf_enabled || ++ __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT)) ++ return 0; ++ ++ /* Allocate a bit more than the initial user buffer for ++ * BPF program. The canonical use case is overriding ++ * TCP_CONGESTION(nv) to TCP_CONGESTION(cubic). ++ */ ++ max_optlen = max_t(int, 16, *optlen); ++ ++ max_optlen = sockopt_alloc_buf(&ctx, max_optlen); ++ if (max_optlen < 0) ++ return max_optlen; ++ ++ ctx.optlen = *optlen; ++ ++ if (copy_from_user(ctx.optval, optval, min(*optlen, max_optlen)) != 0) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ lock_sock(sk); ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT], ++ &ctx, BPF_PROG_RUN); ++ release_sock(sk); ++ ++ if (!ret) { ++ ret = -EPERM; ++ goto out; ++ } ++ ++ if (ctx.optlen == -1) { ++ /* optlen set to -1, bypass kernel */ ++ ret = 1; ++ } else if (ctx.optlen > max_optlen || ctx.optlen < -1) { ++ /* optlen is out of bounds */ ++ ret = -EFAULT; ++ } else { ++ /* optlen within bounds, run kernel handler */ ++ ret = 0; ++ ++ /* export any potential modifications */ ++ *level = ctx.level; ++ *optname = ctx.optname; ++ ++ /* optlen == 0 from BPF indicates that we should ++ * use original userspace data. ++ */ ++ if (ctx.optlen != 0) { ++ *optlen = ctx.optlen; ++ *kernel_optval = ctx.optval; ++ /* export and don't free sockopt buf */ ++ return 0; ++ } ++ } ++ ++out: ++ sockopt_free_buf(&ctx); ++ return ret; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_setsockopt); ++ ++int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, ++ int optname, char __user *optval, ++ int __user *optlen, int max_optlen, ++ int retval) ++{ ++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); ++ struct bpf_sockopt_kern ctx = { ++ .sk = sk, ++ .level = level, ++ .optname = optname, ++ .retval = retval, ++ }; ++ int ret; ++ ++ /* Opportunistic check to see whether we have any BPF program ++ * attached to the hook so we don't waste time allocating ++ * memory and locking the socket. ++ */ ++ if (!cgroup_bpf_enabled || ++ __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT)) ++ return retval; ++ ++ ctx.optlen = max_optlen; ++ ++ max_optlen = sockopt_alloc_buf(&ctx, max_optlen); ++ if (max_optlen < 0) ++ return max_optlen; ++ ++ if (!retval) { ++ /* If kernel getsockopt finished successfully, ++ * copy whatever was returned to the user back ++ * into our temporary buffer. Set optlen to the ++ * one that kernel returned as well to let ++ * BPF programs inspect the value. ++ */ ++ ++ if (get_user(ctx.optlen, optlen)) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ if (ctx.optlen < 0) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ if (copy_from_user(ctx.optval, optval, ++ min(ctx.optlen, max_optlen)) != 0) { ++ ret = -EFAULT; ++ goto out; ++ } ++ } ++ ++ lock_sock(sk); ++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT], ++ &ctx, BPF_PROG_RUN); ++ release_sock(sk); ++ ++ if (!ret) { ++ ret = -EPERM; ++ goto out; ++ } ++ ++ if (ctx.optlen > max_optlen || ctx.optlen < 0) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ /* BPF programs only allowed to set retval to 0, not some ++ * arbitrary value. ++ */ ++ if (ctx.retval != 0 && ctx.retval != retval) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ if (ctx.optlen != 0) { ++ if (copy_to_user(optval, ctx.optval, ctx.optlen) || ++ put_user(ctx.optlen, optlen)) { ++ ret = -EFAULT; ++ goto out; ++ } ++ } ++ ++ ret = ctx.retval; ++ ++out: ++ sockopt_free_buf(&ctx); ++ return ret; ++} ++EXPORT_SYMBOL(__cgroup_bpf_run_filter_getsockopt); ++#endif ++ ++static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp, ++ size_t *lenp) ++{ ++ ssize_t tmp_ret = 0, ret; ++ ++ if (dir->header.parent) { ++ tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp); ++ if (tmp_ret < 0) ++ return tmp_ret; ++ } ++ ++ ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp); ++ if (ret < 0) ++ return ret; ++ *bufp += ret; ++ *lenp -= ret; ++ ret += tmp_ret; ++ ++ /* Avoid leading slash. */ ++ if (!ret) ++ return ret; ++ ++ tmp_ret = strscpy(*bufp, "/", *lenp); ++ if (tmp_ret < 0) ++ return tmp_ret; ++ *bufp += tmp_ret; ++ *lenp -= tmp_ret; ++ ++ return ret + tmp_ret; ++} ++ ++BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf, ++ size_t, buf_len, u64, flags) ++{ ++ ssize_t tmp_ret = 0, ret; ++ ++ if (!buf) ++ return -EINVAL; ++ ++ if (!(flags & BPF_F_SYSCTL_BASE_NAME)) { ++ if (!ctx->head) ++ return -EINVAL; ++ tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len); ++ if (tmp_ret < 0) ++ return tmp_ret; ++ } ++ ++ ret = strscpy(buf, ctx->table->procname, buf_len); ++ ++ return ret < 0 ? ret : tmp_ret + ret; ++} ++ ++static const struct bpf_func_proto bpf_sysctl_get_name_proto = { ++ .func = bpf_sysctl_get_name, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_PTR_TO_MEM, ++ .arg3_type = ARG_CONST_SIZE, ++ .arg4_type = ARG_ANYTHING, ++}; ++ ++static int copy_sysctl_value(char *dst, size_t dst_len, char *src, ++ size_t src_len) ++{ ++ if (!dst) ++ return -EINVAL; ++ ++ if (!dst_len) ++ return -E2BIG; ++ ++ if (!src || !src_len) { ++ memset(dst, 0, dst_len); ++ return -EINVAL; ++ } ++ ++ memcpy(dst, src, min(dst_len, src_len)); ++ ++ if (dst_len > src_len) { ++ memset(dst + src_len, '\0', dst_len - src_len); ++ return src_len; ++ } ++ ++ dst[dst_len - 1] = '\0'; ++ ++ return -E2BIG; ++} ++ ++BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx, ++ char *, buf, size_t, buf_len) ++{ ++ return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len); ++} ++ ++static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { ++ .func = bpf_sysctl_get_current_value, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg3_type = ARG_CONST_SIZE, ++}; ++ ++BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, ++ size_t, buf_len) ++{ ++ if (!ctx->write) { ++ if (buf && buf_len) ++ memset(buf, '\0', buf_len); ++ return -EINVAL; ++ } ++ return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len); ++} ++ ++static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { ++ .func = bpf_sysctl_get_new_value, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg3_type = ARG_CONST_SIZE, ++}; ++ ++BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, ++ const char *, buf, size_t, buf_len) ++{ ++ if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len) ++ return -EINVAL; ++ ++ if (buf_len > PAGE_SIZE - 1) ++ return -E2BIG; ++ ++ memcpy(ctx->new_val, buf, buf_len); ++ ctx->new_len = buf_len; ++ ctx->new_updated = 1; ++ ++ return 0; ++} ++ ++static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { ++ .func = bpf_sysctl_set_new_value, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_PTR_TO_MEM, ++ .arg3_type = ARG_CONST_SIZE, ++}; ++ ++static const struct bpf_func_proto * ++sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ case BPF_FUNC_strtol: ++ return &bpf_strtol_proto; ++ case BPF_FUNC_strtoul: ++ return &bpf_strtoul_proto; ++ case BPF_FUNC_sysctl_get_name: ++ return &bpf_sysctl_get_name_proto; ++ case BPF_FUNC_sysctl_get_current_value: ++ return &bpf_sysctl_get_current_value_proto; ++ case BPF_FUNC_sysctl_get_new_value: ++ return &bpf_sysctl_get_new_value_proto; ++ case BPF_FUNC_sysctl_set_new_value: ++ return &bpf_sysctl_set_new_value_proto; ++ default: ++ return cgroup_base_func_proto(func_id, prog); ++ } ++} ++ ++static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ const int size_default = sizeof(__u32); ++ ++ if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size) ++ return false; ++ ++ switch (off) { ++ case bpf_ctx_range(struct bpf_sysctl, write): ++ if (type != BPF_READ) ++ return false; ++ bpf_ctx_record_field_size(info, size_default); ++ return bpf_ctx_narrow_access_ok(off, size, size_default); ++ case bpf_ctx_range(struct bpf_sysctl, file_pos): ++ if (type == BPF_READ) { ++ bpf_ctx_record_field_size(info, size_default); ++ return bpf_ctx_narrow_access_ok(off, size, size_default); ++ } else { ++ return size == size_default; ++ } ++ default: ++ return false; ++ } ++} ++ ++static u32 sysctl_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ struct bpf_insn *insn = insn_buf; ++ u32 read_size; ++ ++ switch (si->off) { ++ case offsetof(struct bpf_sysctl, write): ++ *insn++ = BPF_LDX_MEM( ++ BPF_SIZE(si->code), si->dst_reg, si->src_reg, ++ bpf_target_off(struct bpf_sysctl_kern, write, ++ FIELD_SIZEOF(struct bpf_sysctl_kern, ++ write), ++ target_size)); ++ break; ++ case offsetof(struct bpf_sysctl, file_pos): ++ /* ppos is a pointer so it should be accessed via indirect ++ * loads and stores. Also for stores additional temporary ++ * register is used since neither src_reg nor dst_reg can be ++ * overridden. ++ */ ++ if (type == BPF_WRITE) { ++ int treg = BPF_REG_9; ++ ++ if (si->src_reg == treg || si->dst_reg == treg) ++ --treg; ++ if (si->src_reg == treg || si->dst_reg == treg) ++ --treg; ++ *insn++ = BPF_STX_MEM( ++ BPF_DW, si->dst_reg, treg, ++ offsetof(struct bpf_sysctl_kern, tmp_reg)); ++ *insn++ = BPF_LDX_MEM( ++ BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos), ++ treg, si->dst_reg, ++ offsetof(struct bpf_sysctl_kern, ppos)); ++ *insn++ = BPF_STX_MEM( ++ BPF_SIZEOF(u32), treg, si->src_reg, ++ bpf_ctx_narrow_access_offset( ++ 0, sizeof(u32), sizeof(loff_t))); ++ *insn++ = BPF_LDX_MEM( ++ BPF_DW, treg, si->dst_reg, ++ offsetof(struct bpf_sysctl_kern, tmp_reg)); ++ } else { ++ *insn++ = BPF_LDX_MEM( ++ BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos), ++ si->dst_reg, si->src_reg, ++ offsetof(struct bpf_sysctl_kern, ppos)); ++ read_size = bpf_size_to_bytes(BPF_SIZE(si->code)); ++ *insn++ = BPF_LDX_MEM( ++ BPF_SIZE(si->code), si->dst_reg, si->dst_reg, ++ bpf_ctx_narrow_access_offset( ++ 0, read_size, sizeof(loff_t))); ++ } ++ *target_size = sizeof(u32); ++ break; ++ } ++ ++ return insn - insn_buf; ++} ++ ++const struct bpf_verifier_ops cg_sysctl_verifier_ops = { ++ .get_func_proto = sysctl_func_proto, ++ .is_valid_access = sysctl_is_valid_access, ++ .convert_ctx_access = sysctl_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops cg_sysctl_prog_ops = { ++}; ++ ++static const struct bpf_func_proto * ++cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++#ifdef CONFIG_NET ++ case BPF_FUNC_sk_storage_get: ++ return &bpf_sk_storage_get_proto; ++ case BPF_FUNC_sk_storage_delete: ++ return &bpf_sk_storage_delete_proto; ++#endif ++#ifdef CONFIG_INET ++ case BPF_FUNC_tcp_sock: ++ return &bpf_tcp_sock_proto; ++#endif ++ default: ++ return cgroup_base_func_proto(func_id, prog); ++ } ++} ++ ++static bool cg_sockopt_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ const int size_default = sizeof(__u32); ++ ++ if (off < 0 || off >= sizeof(struct bpf_sockopt)) ++ return false; ++ ++ if (off % size != 0) ++ return false; ++ ++ if (type == BPF_WRITE) { ++ switch (off) { ++ case offsetof(struct bpf_sockopt, retval): ++ if (size != size_default) ++ return false; ++ return prog->expected_attach_type == ++ BPF_CGROUP_GETSOCKOPT; ++ case offsetof(struct bpf_sockopt, optname): ++ /* fallthrough */ ++ case offsetof(struct bpf_sockopt, level): ++ if (size != size_default) ++ return false; ++ return prog->expected_attach_type == ++ BPF_CGROUP_SETSOCKOPT; ++ case offsetof(struct bpf_sockopt, optlen): ++ return size == size_default; ++ default: ++ return false; ++ } ++ } ++ ++ switch (off) { ++ case offsetof(struct bpf_sockopt, sk): ++ if (size != sizeof(__u64)) ++ return false; ++ info->reg_type = PTR_TO_SOCKET; ++ break; ++ case offsetof(struct bpf_sockopt, optval): ++ if (size != sizeof(__u64)) ++ return false; ++ info->reg_type = PTR_TO_PACKET; ++ break; ++ case offsetof(struct bpf_sockopt, optval_end): ++ if (size != sizeof(__u64)) ++ return false; ++ info->reg_type = PTR_TO_PACKET_END; ++ break; ++ case offsetof(struct bpf_sockopt, retval): ++ if (size != size_default) ++ return false; ++ return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT; ++ default: ++ if (size != size_default) ++ return false; ++ break; ++ } ++ return true; ++} ++ ++#define CG_SOCKOPT_ACCESS_FIELD(T, F) \ ++ T(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F), \ ++ si->dst_reg, si->src_reg, \ ++ offsetof(struct bpf_sockopt_kern, F)) ++ ++static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++{ ++ struct bpf_insn *insn = insn_buf; ++ ++ switch (si->off) { ++ case offsetof(struct bpf_sockopt, sk): ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, sk); ++ break; ++ case offsetof(struct bpf_sockopt, level): ++ if (type == BPF_WRITE) ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, level); ++ else ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, level); ++ break; ++ case offsetof(struct bpf_sockopt, optname): ++ if (type == BPF_WRITE) ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optname); ++ else ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optname); ++ break; ++ case offsetof(struct bpf_sockopt, optlen): ++ if (type == BPF_WRITE) ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optlen); ++ else ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen); ++ break; ++ case offsetof(struct bpf_sockopt, retval): ++ if (type == BPF_WRITE) ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval); ++ else ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval); ++ break; ++ case offsetof(struct bpf_sockopt, optval): ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval); ++ break; ++ case offsetof(struct bpf_sockopt, optval_end): ++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval_end); ++ break; ++ } ++ ++ return insn - insn_buf; ++} ++ ++static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf, ++ bool direct_write, ++ const struct bpf_prog *prog) ++{ ++ /* Nothing to do for sockopt argument. The data is kzalloc'ated. ++ */ ++ return 0; ++} ++ ++const struct bpf_verifier_ops cg_sockopt_verifier_ops = { ++ .get_func_proto = cg_sockopt_func_proto, ++ .is_valid_access = cg_sockopt_is_valid_access, ++ .convert_ctx_access = cg_sockopt_convert_ctx_access, ++ .gen_prologue = cg_sockopt_get_prologue, ++}; ++ ++const struct bpf_prog_ops cg_sockopt_prog_ops = { ++}; +--- a/kernel/bpf/core.c ++++ b/kernel/bpf/core.c +@@ -1,3 +1,4 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later + /* + * Linux Socket Filter - Kernel level socket filtering + * +@@ -12,21 +13,22 @@ + * Alexei Starovoitov + * Daniel Borkmann + * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU General Public License +- * as published by the Free Software Foundation; either version +- * 2 of the License, or (at your option) any later version. +- * + * Andi Kleen - Fix a few bad bugs and races. + * Kris Katterjohn - Added many additional checks in bpf_check_classic() + */ + ++#include + #include + #include + #include + #include + #include + #include ++#include ++#include ++#include ++#include ++#include + + #include + +@@ -47,6 +49,7 @@ + #define DST regs[insn->dst_reg] + #define SRC regs[insn->src_reg] + #define FP regs[BPF_REG_FP] ++#define AX regs[BPF_REG_AX] + #define ARG1 regs[BPF_REG_ARG1] + #define CTX regs[BPF_REG_CTX] + #define IMM insn->imm +@@ -70,10 +73,9 @@ void *bpf_internal_load_pointer_neg_help + return NULL; + } + +-struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) ++struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) + { +- gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | +- gfp_extra_flags; ++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; + struct bpf_prog_aux *aux; + struct bpf_prog *fp; + +@@ -82,8 +84,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned + if (fp == NULL) + return NULL; + +- kmemcheck_annotate_bitfield(fp, meta); +- + aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags); + if (aux == NULL) { + vfree(fp); +@@ -93,30 +93,151 @@ struct bpf_prog *bpf_prog_alloc(unsigned + fp->pages = size / PAGE_SIZE; + fp->aux = aux; + fp->aux->prog = fp; ++ fp->jit_requested = ebpf_jit_enabled(); ++ ++ INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode); + + return fp; + } ++ ++struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) ++{ ++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; ++ struct bpf_prog *prog; ++ int cpu; ++ ++ prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); ++ if (!prog) ++ return NULL; ++ ++ prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); ++ if (!prog->aux->stats) { ++ kfree(prog->aux); ++ vfree(prog); ++ return NULL; ++ } ++ ++ for_each_possible_cpu(cpu) { ++ struct bpf_prog_stats *pstats; ++ ++ pstats = per_cpu_ptr(prog->aux->stats, cpu); ++ u64_stats_init(&pstats->syncp); ++ } ++ return prog; ++} + EXPORT_SYMBOL_GPL(bpf_prog_alloc); + ++int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) ++{ ++ if (!prog->aux->nr_linfo || !prog->jit_requested) ++ return 0; ++ ++ prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, ++ sizeof(*prog->aux->jited_linfo), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!prog->aux->jited_linfo) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++void bpf_prog_free_jited_linfo(struct bpf_prog *prog) ++{ ++ kfree(prog->aux->jited_linfo); ++ prog->aux->jited_linfo = NULL; ++} ++ ++void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog) ++{ ++ if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) ++ bpf_prog_free_jited_linfo(prog); ++} ++ ++/* The jit engine is responsible to provide an array ++ * for insn_off to the jited_off mapping (insn_to_jit_off). ++ * ++ * The idx to this array is the insn_off. Hence, the insn_off ++ * here is relative to the prog itself instead of the main prog. ++ * This array has one entry for each xlated bpf insn. ++ * ++ * jited_off is the byte off to the last byte of the jited insn. ++ * ++ * Hence, with ++ * insn_start: ++ * The first bpf insn off of the prog. The insn off ++ * here is relative to the main prog. ++ * e.g. if prog is a subprog, insn_start > 0 ++ * linfo_idx: ++ * The prog's idx to prog->aux->linfo and jited_linfo ++ * ++ * jited_linfo[linfo_idx] = prog->bpf_func ++ * ++ * For i > linfo_idx, ++ * ++ * jited_linfo[i] = prog->bpf_func + ++ * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] ++ */ ++void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, ++ const u32 *insn_to_jit_off) ++{ ++ u32 linfo_idx, insn_start, insn_end, nr_linfo, i; ++ const struct bpf_line_info *linfo; ++ void **jited_linfo; ++ ++ if (!prog->aux->jited_linfo) ++ /* Userspace did not provide linfo */ ++ return; ++ ++ linfo_idx = prog->aux->linfo_idx; ++ linfo = &prog->aux->linfo[linfo_idx]; ++ insn_start = linfo[0].insn_off; ++ insn_end = insn_start + prog->len; ++ ++ jited_linfo = &prog->aux->jited_linfo[linfo_idx]; ++ jited_linfo[0] = prog->bpf_func; ++ ++ nr_linfo = prog->aux->nr_linfo - linfo_idx; ++ ++ for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) ++ /* The verifier ensures that linfo[i].insn_off is ++ * strictly increasing ++ */ ++ jited_linfo[i] = prog->bpf_func + ++ insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; ++} ++ ++void bpf_prog_free_linfo(struct bpf_prog *prog) ++{ ++ bpf_prog_free_jited_linfo(prog); ++ kvfree(prog->aux->linfo); ++} ++ + struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, + gfp_t gfp_extra_flags) + { +- gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | +- gfp_extra_flags; ++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; + struct bpf_prog *fp; ++ u32 pages, delta; ++ int ret; + + BUG_ON(fp_old == NULL); + + size = round_up(size, PAGE_SIZE); +- if (size <= fp_old->pages * PAGE_SIZE) ++ pages = size / PAGE_SIZE; ++ if (pages <= fp_old->pages) + return fp_old; + +- fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); +- if (fp != NULL) { +- kmemcheck_annotate_bitfield(fp, meta); ++ delta = pages - fp_old->pages; ++ ret = __bpf_prog_charge(fp_old->aux->user, delta); ++ if (ret) ++ return NULL; + ++ fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); ++ if (fp == NULL) { ++ __bpf_prog_uncharge(fp_old->aux->user, delta); ++ } else { + memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); +- fp->pages = size / PAGE_SIZE; ++ fp->pages = pages; + fp->aux->prog = fp; + + /* We keep fp->aux from fp_old around in the new +@@ -128,40 +249,578 @@ struct bpf_prog *bpf_prog_realloc(struct + + return fp; + } +-EXPORT_SYMBOL_GPL(bpf_prog_realloc); + + void __bpf_prog_free(struct bpf_prog *fp) + { +- kfree(fp->aux); ++ if (fp->aux) { ++ free_percpu(fp->aux->stats); ++ kfree(fp->aux); ++ } + vfree(fp); + } +-EXPORT_SYMBOL_GPL(__bpf_prog_free); ++ ++int bpf_prog_calc_tag(struct bpf_prog *fp) ++{ ++ const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64); ++ u32 raw_size = bpf_prog_tag_scratch_size(fp); ++ u32 digest[SHA_DIGEST_WORDS]; ++ u32 ws[SHA_WORKSPACE_WORDS]; ++ u32 i, bsize, psize, blocks; ++ struct bpf_insn *dst; ++ bool was_ld_map; ++ u8 *raw, *todo; ++ __be32 *result; ++ __be64 *bits; ++ ++ raw = vmalloc(raw_size); ++ if (!raw) ++ return -ENOMEM; ++ ++ sha_init(digest); ++ memset(ws, 0, sizeof(ws)); ++ ++ /* We need to take out the map fd for the digest calculation ++ * since they are unstable from user space side. ++ */ ++ dst = (void *)raw; ++ for (i = 0, was_ld_map = false; i < fp->len; i++) { ++ dst[i] = fp->insnsi[i]; ++ if (!was_ld_map && ++ dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && ++ (dst[i].src_reg == BPF_PSEUDO_MAP_FD || ++ dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { ++ was_ld_map = true; ++ dst[i].imm = 0; ++ } else if (was_ld_map && ++ dst[i].code == 0 && ++ dst[i].dst_reg == 0 && ++ dst[i].src_reg == 0 && ++ dst[i].off == 0) { ++ was_ld_map = false; ++ dst[i].imm = 0; ++ } else { ++ was_ld_map = false; ++ } ++ } ++ ++ psize = bpf_prog_insn_size(fp); ++ memset(&raw[psize], 0, raw_size - psize); ++ raw[psize++] = 0x80; ++ ++ bsize = round_up(psize, SHA_MESSAGE_BYTES); ++ blocks = bsize / SHA_MESSAGE_BYTES; ++ todo = raw; ++ if (bsize - psize >= sizeof(__be64)) { ++ bits = (__be64 *)(todo + bsize - sizeof(__be64)); ++ } else { ++ bits = (__be64 *)(todo + bsize + bits_offset); ++ blocks++; ++ } ++ *bits = cpu_to_be64((psize - 1) << 3); ++ ++ while (blocks--) { ++ sha_transform(digest, todo, ws); ++ todo += SHA_MESSAGE_BYTES; ++ } ++ ++ result = (__force __be32 *)digest; ++ for (i = 0; i < SHA_DIGEST_WORDS; i++) ++ result[i] = cpu_to_be32(digest[i]); ++ memcpy(fp->tag, result, sizeof(fp->tag)); ++ ++ vfree(raw); ++ return 0; ++} ++ ++static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, ++ s32 end_new, s32 curr, const bool probe_pass) ++{ ++ const s64 imm_min = S32_MIN, imm_max = S32_MAX; ++ s32 delta = end_new - end_old; ++ s64 imm = insn->imm; ++ ++ if (curr < pos && curr + imm + 1 >= end_old) ++ imm += delta; ++ else if (curr >= end_new && curr + imm + 1 < end_new) ++ imm -= delta; ++ if (imm < imm_min || imm > imm_max) ++ return -ERANGE; ++ if (!probe_pass) ++ insn->imm = imm; ++ return 0; ++} ++ ++static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, ++ s32 end_new, s32 curr, const bool probe_pass) ++{ ++ const s32 off_min = S16_MIN, off_max = S16_MAX; ++ s32 delta = end_new - end_old; ++ s32 off = insn->off; ++ ++ if (curr < pos && curr + off + 1 >= end_old) ++ off += delta; ++ else if (curr >= end_new && curr + off + 1 < end_new) ++ off -= delta; ++ if (off < off_min || off > off_max) ++ return -ERANGE; ++ if (!probe_pass) ++ insn->off = off; ++ return 0; ++} ++ ++static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, ++ s32 end_new, const bool probe_pass) ++{ ++ u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); ++ struct bpf_insn *insn = prog->insnsi; ++ int ret = 0; ++ ++ for (i = 0; i < insn_cnt; i++, insn++) { ++ u8 code; ++ ++ /* In the probing pass we still operate on the original, ++ * unpatched image in order to check overflows before we ++ * do any other adjustments. Therefore skip the patchlet. ++ */ ++ if (probe_pass && i == pos) { ++ i = end_new; ++ insn = prog->insnsi + end_old; ++ } ++ code = insn->code; ++ if ((BPF_CLASS(code) != BPF_JMP && ++ BPF_CLASS(code) != BPF_JMP32) || ++ BPF_OP(code) == BPF_EXIT) ++ continue; ++ /* Adjust offset of jmps if we cross patch boundaries. */ ++ if (BPF_OP(code) == BPF_CALL) { ++ if (insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ ret = bpf_adj_delta_to_imm(insn, pos, end_old, ++ end_new, i, probe_pass); ++ } else { ++ ret = bpf_adj_delta_to_off(insn, pos, end_old, ++ end_new, i, probe_pass); ++ } ++ if (ret) ++ break; ++ } ++ ++ return ret; ++} ++ ++static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) ++{ ++ struct bpf_line_info *linfo; ++ u32 i, nr_linfo; ++ ++ nr_linfo = prog->aux->nr_linfo; ++ if (!nr_linfo || !delta) ++ return; ++ ++ linfo = prog->aux->linfo; ++ ++ for (i = 0; i < nr_linfo; i++) ++ if (off < linfo[i].insn_off) ++ break; ++ ++ /* Push all off < linfo[i].insn_off by delta */ ++ for (; i < nr_linfo; i++) ++ linfo[i].insn_off += delta; ++} ++ ++struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, ++ const struct bpf_insn *patch, u32 len) ++{ ++ u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; ++ const u32 cnt_max = S16_MAX; ++ struct bpf_prog *prog_adj; ++ int err; ++ ++ /* Since our patchlet doesn't expand the image, we're done. */ ++ if (insn_delta == 0) { ++ memcpy(prog->insnsi + off, patch, sizeof(*patch)); ++ return prog; ++ } ++ ++ insn_adj_cnt = prog->len + insn_delta; ++ ++ /* Reject anything that would potentially let the insn->off ++ * target overflow when we have excessive program expansions. ++ * We need to probe here before we do any reallocation where ++ * we afterwards may not fail anymore. ++ */ ++ if (insn_adj_cnt > cnt_max && ++ (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) ++ return ERR_PTR(err); ++ ++ /* Several new instructions need to be inserted. Make room ++ * for them. Likely, there's no need for a new allocation as ++ * last page could have large enough tailroom. ++ */ ++ prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), ++ GFP_USER); ++ if (!prog_adj) ++ return ERR_PTR(-ENOMEM); ++ ++ prog_adj->len = insn_adj_cnt; ++ ++ /* Patching happens in 3 steps: ++ * ++ * 1) Move over tail of insnsi from next instruction onwards, ++ * so we can patch the single target insn with one or more ++ * new ones (patching is always from 1 to n insns, n > 0). ++ * 2) Inject new instructions at the target location. ++ * 3) Adjust branch offsets if necessary. ++ */ ++ insn_rest = insn_adj_cnt - off - len; ++ ++ memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, ++ sizeof(*patch) * insn_rest); ++ memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); ++ ++ /* We are guaranteed to not fail at this point, otherwise ++ * the ship has sailed to reverse to the original state. An ++ * overflow cannot happen at this point. ++ */ ++ BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); ++ ++ bpf_adj_linfo(prog_adj, off, insn_delta); ++ ++ return prog_adj; ++} ++ ++int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) ++{ ++ /* Branch offsets can't overflow when program is shrinking, no need ++ * to call bpf_adj_branches(..., true) here ++ */ ++ memmove(prog->insnsi + off, prog->insnsi + off + cnt, ++ sizeof(struct bpf_insn) * (prog->len - off - cnt)); ++ prog->len -= cnt; ++ ++ return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); ++} ++ ++static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) ++{ ++ int i; ++ ++ for (i = 0; i < fp->aux->func_cnt; i++) ++ bpf_prog_kallsyms_del(fp->aux->func[i]); ++} ++ ++void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) ++{ ++ bpf_prog_kallsyms_del_subprogs(fp); ++ bpf_prog_kallsyms_del(fp); ++} + + #ifdef CONFIG_BPF_JIT ++/* All BPF JIT sysctl knobs here. */ ++int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON); ++int bpf_jit_harden __read_mostly; ++int bpf_jit_kallsyms __read_mostly; ++long bpf_jit_limit __read_mostly; ++ ++static __always_inline void ++bpf_get_prog_addr_region(const struct bpf_prog *prog, ++ unsigned long *symbol_start, ++ unsigned long *symbol_end) ++{ ++ const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog); ++ unsigned long addr = (unsigned long)hdr; ++ ++ WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); ++ ++ *symbol_start = addr; ++ *symbol_end = addr + hdr->pages * PAGE_SIZE; ++} ++ ++void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) ++{ ++ const char *end = sym + KSYM_NAME_LEN; ++ const struct btf_type *type; ++ const char *func_name; ++ ++ BUILD_BUG_ON(sizeof("bpf_prog_") + ++ sizeof(prog->tag) * 2 + ++ /* name has been null terminated. ++ * We should need +1 for the '_' preceding ++ * the name. However, the null character ++ * is double counted between the name and the ++ * sizeof("bpf_prog_") above, so we omit ++ * the +1 here. ++ */ ++ sizeof(prog->aux->name) > KSYM_NAME_LEN); ++ ++ sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); ++ sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); ++ ++ /* prog->aux->name will be ignored if full btf name is available */ ++ if (prog->aux->func_info_cnt) { ++ type = btf_type_by_id(prog->aux->btf, ++ prog->aux->func_info[prog->aux->func_idx].type_id); ++ func_name = btf_name_by_offset(prog->aux->btf, type->name_off); ++ snprintf(sym, (size_t)(end - sym), "_%s", func_name); ++ return; ++ } ++ ++ if (prog->aux->name[0]) ++ snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); ++ else ++ *sym = 0; ++} ++ ++static __always_inline unsigned long ++bpf_get_prog_addr_start(struct latch_tree_node *n) ++{ ++ unsigned long symbol_start, symbol_end; ++ const struct bpf_prog_aux *aux; ++ ++ aux = container_of(n, struct bpf_prog_aux, ksym_tnode); ++ bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); ++ ++ return symbol_start; ++} ++ ++static __always_inline bool bpf_tree_less(struct latch_tree_node *a, ++ struct latch_tree_node *b) ++{ ++ return bpf_get_prog_addr_start(a) < bpf_get_prog_addr_start(b); ++} ++ ++static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) ++{ ++ unsigned long val = (unsigned long)key; ++ unsigned long symbol_start, symbol_end; ++ const struct bpf_prog_aux *aux; ++ ++ aux = container_of(n, struct bpf_prog_aux, ksym_tnode); ++ bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); ++ ++ if (val < symbol_start) ++ return -1; ++ if (val >= symbol_end) ++ return 1; ++ ++ return 0; ++} ++ ++static const struct latch_tree_ops bpf_tree_ops = { ++ .less = bpf_tree_less, ++ .comp = bpf_tree_comp, ++}; ++ ++static DEFINE_SPINLOCK(bpf_lock); ++static LIST_HEAD(bpf_kallsyms); ++static struct latch_tree_root bpf_tree __cacheline_aligned; ++ ++static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) ++{ ++ WARN_ON_ONCE(!list_empty(&aux->ksym_lnode)); ++ list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms); ++ latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); ++} ++ ++static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux) ++{ ++ if (list_empty(&aux->ksym_lnode)) ++ return; ++ ++ latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); ++ list_del_rcu(&aux->ksym_lnode); ++} ++ ++static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) ++{ ++ return fp->jited && !bpf_prog_was_classic(fp); ++} ++ ++static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) ++{ ++ return list_empty(&fp->aux->ksym_lnode) || ++ fp->aux->ksym_lnode.prev == LIST_POISON2; ++} ++ ++void bpf_prog_kallsyms_add(struct bpf_prog *fp) ++{ ++ if (!bpf_prog_kallsyms_candidate(fp) || ++ !capable(CAP_SYS_ADMIN)) ++ return; ++ ++ spin_lock_bh(&bpf_lock); ++ bpf_prog_ksym_node_add(fp->aux); ++ spin_unlock_bh(&bpf_lock); ++} ++ ++void bpf_prog_kallsyms_del(struct bpf_prog *fp) ++{ ++ if (!bpf_prog_kallsyms_candidate(fp)) ++ return; ++ ++ spin_lock_bh(&bpf_lock); ++ bpf_prog_ksym_node_del(fp->aux); ++ spin_unlock_bh(&bpf_lock); ++} ++ ++static struct bpf_prog *bpf_prog_kallsyms_find(unsigned long addr) ++{ ++ struct latch_tree_node *n; ++ ++ if (!bpf_jit_kallsyms_enabled()) ++ return NULL; ++ ++ n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); ++ return n ? ++ container_of(n, struct bpf_prog_aux, ksym_tnode)->prog : ++ NULL; ++} ++ ++const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, ++ unsigned long *off, char *sym) ++{ ++ unsigned long symbol_start, symbol_end; ++ struct bpf_prog *prog; ++ char *ret = NULL; ++ ++ rcu_read_lock(); ++ prog = bpf_prog_kallsyms_find(addr); ++ if (prog) { ++ bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end); ++ bpf_get_prog_name(prog, sym); ++ ++ ret = sym; ++ if (size) ++ *size = symbol_end - symbol_start; ++ if (off) ++ *off = addr - symbol_start; ++ } ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++bool is_bpf_text_address(unsigned long addr) ++{ ++ bool ret; ++ ++ rcu_read_lock(); ++ ret = bpf_prog_kallsyms_find(addr) != NULL; ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, ++ char *sym) ++{ ++ struct bpf_prog_aux *aux; ++ unsigned int it = 0; ++ int ret = -ERANGE; ++ ++ if (!bpf_jit_kallsyms_enabled()) ++ return ret; ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) { ++ if (it++ != symnum) ++ continue; ++ ++ bpf_get_prog_name(aux->prog, sym); ++ ++ *value = (unsigned long)aux->prog->bpf_func; ++ *type = BPF_SYM_ELF_TYPE; ++ ++ ret = 0; ++ break; ++ } ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static atomic_long_t bpf_jit_current; ++ ++/* Can be overridden by an arch's JIT compiler if it has a custom, ++ * dedicated BPF backend memory area, or if neither of the two ++ * below apply. ++ */ ++u64 __weak bpf_jit_alloc_exec_limit(void) ++{ ++#if defined(MODULES_VADDR) ++ return MODULES_END - MODULES_VADDR; ++#else ++ return VMALLOC_END - VMALLOC_START; ++#endif ++} ++ ++static int __init bpf_jit_charge_init(void) ++{ ++ /* Only used as heuristic here to derive limit. */ ++ bpf_jit_limit = min_t(u64, round_up(bpf_jit_alloc_exec_limit() >> 2, ++ PAGE_SIZE), LONG_MAX); ++ return 0; ++} ++pure_initcall(bpf_jit_charge_init); ++ ++static int bpf_jit_charge_modmem(u32 pages) ++{ ++ if (atomic_long_add_return(pages, &bpf_jit_current) > ++ (bpf_jit_limit >> PAGE_SHIFT)) { ++ if (!capable(CAP_SYS_ADMIN)) { ++ atomic_long_sub(pages, &bpf_jit_current); ++ return -EPERM; ++ } ++ } ++ ++ return 0; ++} ++ ++static void bpf_jit_uncharge_modmem(u32 pages) ++{ ++ atomic_long_sub(pages, &bpf_jit_current); ++} ++ ++void *__weak bpf_jit_alloc_exec(unsigned long size) ++{ ++ return module_alloc(size); ++} ++ ++void __weak bpf_jit_free_exec(void *addr) ++{ ++ module_memfree(addr); ++} ++ + struct bpf_binary_header * + bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, + unsigned int alignment, + bpf_jit_fill_hole_t bpf_fill_ill_insns) + { + struct bpf_binary_header *hdr; +- unsigned int size, hole, start; ++ u32 size, hole, start, pages; + + /* Most of BPF filters are really small, but if some of them + * fill a page, allow at least 128 extra bytes to insert a + * random section of illegal instructions. + */ + size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); +- hdr = module_alloc(size); +- if (hdr == NULL) ++ pages = size / PAGE_SIZE; ++ ++ if (bpf_jit_charge_modmem(pages)) ++ return NULL; ++ hdr = bpf_jit_alloc_exec(size); ++ if (!hdr) { ++ bpf_jit_uncharge_modmem(pages); + return NULL; ++ } + + /* Fill space with illegal/arch-dep instructions. */ + bpf_fill_ill_insns(hdr, size); + +- hdr->pages = size / PAGE_SIZE; ++ hdr->pages = pages; + hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), + PAGE_SIZE - sizeof(*hdr)); +- start = (prandom_u32() % hole) & ~(alignment - 1); ++ start = (get_random_int() % hole) & ~(alignment - 1); + + /* Leave a random number of instructions before BPF code. */ + *image_ptr = &hdr->image[start]; +@@ -171,13 +830,301 @@ bpf_jit_binary_alloc(unsigned int progle + + void bpf_jit_binary_free(struct bpf_binary_header *hdr) + { +- module_memfree(hdr); ++ u32 pages = hdr->pages; ++ ++ bpf_jit_free_exec(hdr); ++ bpf_jit_uncharge_modmem(pages); ++} ++ ++/* This symbol is only overridden by archs that have different ++ * requirements than the usual eBPF JITs, f.e. when they only ++ * implement cBPF JIT, do not set images read-only, etc. ++ */ ++void __weak bpf_jit_free(struct bpf_prog *fp) ++{ ++ if (fp->jited) { ++ struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); ++ ++ bpf_jit_binary_unlock_ro(hdr); ++ bpf_jit_binary_free(hdr); ++ ++ WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); ++ } ++ ++ bpf_prog_unlock_free(fp); ++} ++ ++int bpf_jit_get_func_addr(const struct bpf_prog *prog, ++ const struct bpf_insn *insn, bool extra_pass, ++ u64 *func_addr, bool *func_addr_fixed) ++{ ++ s16 off = insn->off; ++ s32 imm = insn->imm; ++ u8 *addr; ++ ++ *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; ++ if (!*func_addr_fixed) { ++ /* Place-holder address till the last pass has collected ++ * all addresses for JITed subprograms in which case we ++ * can pick them up from prog->aux. ++ */ ++ if (!extra_pass) ++ addr = NULL; ++ else if (prog->aux->func && ++ off >= 0 && off < prog->aux->func_cnt) ++ addr = (u8 *)prog->aux->func[off]->bpf_func; ++ else ++ return -EINVAL; ++ } else { ++ /* Address of a BPF helper call. Since part of the core ++ * kernel, it's always at a fixed location. __bpf_call_base ++ * and the helper with imm relative to it are both in core ++ * kernel. ++ */ ++ addr = (u8 *)__bpf_call_base + imm; ++ } ++ ++ *func_addr = (unsigned long)addr; ++ return 0; ++} ++ ++static int bpf_jit_blind_insn(const struct bpf_insn *from, ++ const struct bpf_insn *aux, ++ struct bpf_insn *to_buff, ++ bool emit_zext) ++{ ++ struct bpf_insn *to = to_buff; ++ u32 imm_rnd = get_random_int(); ++ s16 off; ++ ++ BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); ++ BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); ++ ++ /* Constraints on AX register: ++ * ++ * AX register is inaccessible from user space. It is mapped in ++ * all JITs, and used here for constant blinding rewrites. It is ++ * typically "stateless" meaning its contents are only valid within ++ * the executed instruction, but not across several instructions. ++ * There are a few exceptions however which are further detailed ++ * below. ++ * ++ * Constant blinding is only used by JITs, not in the interpreter. ++ * The interpreter uses AX in some occasions as a local temporary ++ * register e.g. in DIV or MOD instructions. ++ * ++ * In restricted circumstances, the verifier can also use the AX ++ * register for rewrites as long as they do not interfere with ++ * the above cases! ++ */ ++ if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) ++ goto out; ++ ++ if (from->imm == 0 && ++ (from->code == (BPF_ALU | BPF_MOV | BPF_K) || ++ from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { ++ *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); ++ goto out; ++ } ++ ++ switch (from->code) { ++ case BPF_ALU | BPF_ADD | BPF_K: ++ case BPF_ALU | BPF_SUB | BPF_K: ++ case BPF_ALU | BPF_AND | BPF_K: ++ case BPF_ALU | BPF_OR | BPF_K: ++ case BPF_ALU | BPF_XOR | BPF_K: ++ case BPF_ALU | BPF_MUL | BPF_K: ++ case BPF_ALU | BPF_MOV | BPF_K: ++ case BPF_ALU | BPF_DIV | BPF_K: ++ case BPF_ALU | BPF_MOD | BPF_K: ++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); ++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX); ++ break; ++ ++ case BPF_ALU64 | BPF_ADD | BPF_K: ++ case BPF_ALU64 | BPF_SUB | BPF_K: ++ case BPF_ALU64 | BPF_AND | BPF_K: ++ case BPF_ALU64 | BPF_OR | BPF_K: ++ case BPF_ALU64 | BPF_XOR | BPF_K: ++ case BPF_ALU64 | BPF_MUL | BPF_K: ++ case BPF_ALU64 | BPF_MOV | BPF_K: ++ case BPF_ALU64 | BPF_DIV | BPF_K: ++ case BPF_ALU64 | BPF_MOD | BPF_K: ++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); ++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX); ++ break; ++ ++ case BPF_JMP | BPF_JEQ | BPF_K: ++ case BPF_JMP | BPF_JNE | BPF_K: ++ case BPF_JMP | BPF_JGT | BPF_K: ++ case BPF_JMP | BPF_JLT | BPF_K: ++ case BPF_JMP | BPF_JGE | BPF_K: ++ case BPF_JMP | BPF_JLE | BPF_K: ++ case BPF_JMP | BPF_JSGT | BPF_K: ++ case BPF_JMP | BPF_JSLT | BPF_K: ++ case BPF_JMP | BPF_JSGE | BPF_K: ++ case BPF_JMP | BPF_JSLE | BPF_K: ++ case BPF_JMP | BPF_JSET | BPF_K: ++ /* Accommodate for extra offset in case of a backjump. */ ++ off = from->off; ++ if (off < 0) ++ off -= 2; ++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); ++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); ++ break; ++ ++ case BPF_JMP32 | BPF_JEQ | BPF_K: ++ case BPF_JMP32 | BPF_JNE | BPF_K: ++ case BPF_JMP32 | BPF_JGT | BPF_K: ++ case BPF_JMP32 | BPF_JLT | BPF_K: ++ case BPF_JMP32 | BPF_JGE | BPF_K: ++ case BPF_JMP32 | BPF_JLE | BPF_K: ++ case BPF_JMP32 | BPF_JSGT | BPF_K: ++ case BPF_JMP32 | BPF_JSLT | BPF_K: ++ case BPF_JMP32 | BPF_JSGE | BPF_K: ++ case BPF_JMP32 | BPF_JSLE | BPF_K: ++ case BPF_JMP32 | BPF_JSET | BPF_K: ++ /* Accommodate for extra offset in case of a backjump. */ ++ off = from->off; ++ if (off < 0) ++ off -= 2; ++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); ++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, ++ off); ++ break; ++ ++ case BPF_LD | BPF_IMM | BPF_DW: ++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); ++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); ++ *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); ++ break; ++ case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ ++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); ++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ if (emit_zext) ++ *to++ = BPF_ZEXT_REG(BPF_REG_AX); ++ *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); ++ break; ++ ++ case BPF_ST | BPF_MEM | BPF_DW: ++ case BPF_ST | BPF_MEM | BPF_W: ++ case BPF_ST | BPF_MEM | BPF_H: ++ case BPF_ST | BPF_MEM | BPF_B: ++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); ++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); ++ *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); ++ break; ++ } ++out: ++ return to - to_buff; ++} ++ ++static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, ++ gfp_t gfp_extra_flags) ++{ ++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; ++ struct bpf_prog *fp; ++ ++ fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL); ++ if (fp != NULL) { ++ /* aux->prog still points to the fp_other one, so ++ * when promoting the clone to the real program, ++ * this still needs to be adapted. ++ */ ++ memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); ++ } ++ ++ return fp; ++} ++ ++static void bpf_prog_clone_free(struct bpf_prog *fp) ++{ ++ /* aux was stolen by the other clone, so we cannot free ++ * it from this path! It will be freed eventually by the ++ * other program on release. ++ * ++ * At this point, we don't need a deferred release since ++ * clone is guaranteed to not be locked. ++ */ ++ fp->aux = NULL; ++ __bpf_prog_free(fp); ++} ++ ++void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) ++{ ++ /* We have to repoint aux->prog to self, as we don't ++ * know whether fp here is the clone or the original. ++ */ ++ fp->aux->prog = fp; ++ bpf_prog_clone_free(fp_other); ++} ++ ++struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) ++{ ++ struct bpf_insn insn_buff[16], aux[2]; ++ struct bpf_prog *clone, *tmp; ++ int insn_delta, insn_cnt; ++ struct bpf_insn *insn; ++ int i, rewritten; ++ ++ if (!bpf_jit_blinding_enabled(prog) || prog->blinded) ++ return prog; ++ ++ clone = bpf_prog_clone_create(prog, GFP_USER); ++ if (!clone) ++ return ERR_PTR(-ENOMEM); ++ ++ insn_cnt = clone->len; ++ insn = clone->insnsi; ++ ++ for (i = 0; i < insn_cnt; i++, insn++) { ++ /* We temporarily need to hold the original ld64 insn ++ * so that we can still access the first part in the ++ * second blinding run. ++ */ ++ if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && ++ insn[1].code == 0) ++ memcpy(aux, insn, sizeof(aux)); ++ ++ rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, ++ clone->aux->verifier_zext); ++ if (!rewritten) ++ continue; ++ ++ tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); ++ if (IS_ERR(tmp)) { ++ /* Patching may have repointed aux->prog during ++ * realloc from the original one, so we need to ++ * fix it up here on error. ++ */ ++ bpf_jit_prog_release_other(prog, clone); ++ return tmp; ++ } ++ ++ clone = tmp; ++ insn_delta = rewritten - 1; ++ ++ /* Walk new program and skip insns we just inserted. */ ++ insn = clone->insnsi + i + insn_delta; ++ insn_cnt += insn_delta; ++ i += insn_delta; ++ } ++ ++ clone->blinded = 1; ++ return clone; + } + #endif /* CONFIG_BPF_JIT */ + + /* Base function for offset calculation. Needs to go into .text section, + * therefore keeping it non-static as well; will also be used by JITs +- * anyway later on, so do not let the compiler omit it. ++ * anyway later on, so do not let the compiler omit it. This also needs ++ * to go into kallsyms for correlation from e.g. bpftool, so naming ++ * must not change. + */ + noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) + { +@@ -185,157 +1132,243 @@ noinline u64 __bpf_call_base(u64 r1, u64 + } + EXPORT_SYMBOL_GPL(__bpf_call_base); + ++/* All UAPI available opcodes. */ ++#define BPF_INSN_MAP(INSN_2, INSN_3) \ ++ /* 32 bit ALU operations. */ \ ++ /* Register based. */ \ ++ INSN_3(ALU, ADD, X), \ ++ INSN_3(ALU, SUB, X), \ ++ INSN_3(ALU, AND, X), \ ++ INSN_3(ALU, OR, X), \ ++ INSN_3(ALU, LSH, X), \ ++ INSN_3(ALU, RSH, X), \ ++ INSN_3(ALU, XOR, X), \ ++ INSN_3(ALU, MUL, X), \ ++ INSN_3(ALU, MOV, X), \ ++ INSN_3(ALU, ARSH, X), \ ++ INSN_3(ALU, DIV, X), \ ++ INSN_3(ALU, MOD, X), \ ++ INSN_2(ALU, NEG), \ ++ INSN_3(ALU, END, TO_BE), \ ++ INSN_3(ALU, END, TO_LE), \ ++ /* Immediate based. */ \ ++ INSN_3(ALU, ADD, K), \ ++ INSN_3(ALU, SUB, K), \ ++ INSN_3(ALU, AND, K), \ ++ INSN_3(ALU, OR, K), \ ++ INSN_3(ALU, LSH, K), \ ++ INSN_3(ALU, RSH, K), \ ++ INSN_3(ALU, XOR, K), \ ++ INSN_3(ALU, MUL, K), \ ++ INSN_3(ALU, MOV, K), \ ++ INSN_3(ALU, ARSH, K), \ ++ INSN_3(ALU, DIV, K), \ ++ INSN_3(ALU, MOD, K), \ ++ /* 64 bit ALU operations. */ \ ++ /* Register based. */ \ ++ INSN_3(ALU64, ADD, X), \ ++ INSN_3(ALU64, SUB, X), \ ++ INSN_3(ALU64, AND, X), \ ++ INSN_3(ALU64, OR, X), \ ++ INSN_3(ALU64, LSH, X), \ ++ INSN_3(ALU64, RSH, X), \ ++ INSN_3(ALU64, XOR, X), \ ++ INSN_3(ALU64, MUL, X), \ ++ INSN_3(ALU64, MOV, X), \ ++ INSN_3(ALU64, ARSH, X), \ ++ INSN_3(ALU64, DIV, X), \ ++ INSN_3(ALU64, MOD, X), \ ++ INSN_2(ALU64, NEG), \ ++ /* Immediate based. */ \ ++ INSN_3(ALU64, ADD, K), \ ++ INSN_3(ALU64, SUB, K), \ ++ INSN_3(ALU64, AND, K), \ ++ INSN_3(ALU64, OR, K), \ ++ INSN_3(ALU64, LSH, K), \ ++ INSN_3(ALU64, RSH, K), \ ++ INSN_3(ALU64, XOR, K), \ ++ INSN_3(ALU64, MUL, K), \ ++ INSN_3(ALU64, MOV, K), \ ++ INSN_3(ALU64, ARSH, K), \ ++ INSN_3(ALU64, DIV, K), \ ++ INSN_3(ALU64, MOD, K), \ ++ /* Call instruction. */ \ ++ INSN_2(JMP, CALL), \ ++ /* Exit instruction. */ \ ++ INSN_2(JMP, EXIT), \ ++ /* 32-bit Jump instructions. */ \ ++ /* Register based. */ \ ++ INSN_3(JMP32, JEQ, X), \ ++ INSN_3(JMP32, JNE, X), \ ++ INSN_3(JMP32, JGT, X), \ ++ INSN_3(JMP32, JLT, X), \ ++ INSN_3(JMP32, JGE, X), \ ++ INSN_3(JMP32, JLE, X), \ ++ INSN_3(JMP32, JSGT, X), \ ++ INSN_3(JMP32, JSLT, X), \ ++ INSN_3(JMP32, JSGE, X), \ ++ INSN_3(JMP32, JSLE, X), \ ++ INSN_3(JMP32, JSET, X), \ ++ /* Immediate based. */ \ ++ INSN_3(JMP32, JEQ, K), \ ++ INSN_3(JMP32, JNE, K), \ ++ INSN_3(JMP32, JGT, K), \ ++ INSN_3(JMP32, JLT, K), \ ++ INSN_3(JMP32, JGE, K), \ ++ INSN_3(JMP32, JLE, K), \ ++ INSN_3(JMP32, JSGT, K), \ ++ INSN_3(JMP32, JSLT, K), \ ++ INSN_3(JMP32, JSGE, K), \ ++ INSN_3(JMP32, JSLE, K), \ ++ INSN_3(JMP32, JSET, K), \ ++ /* Jump instructions. */ \ ++ /* Register based. */ \ ++ INSN_3(JMP, JEQ, X), \ ++ INSN_3(JMP, JNE, X), \ ++ INSN_3(JMP, JGT, X), \ ++ INSN_3(JMP, JLT, X), \ ++ INSN_3(JMP, JGE, X), \ ++ INSN_3(JMP, JLE, X), \ ++ INSN_3(JMP, JSGT, X), \ ++ INSN_3(JMP, JSLT, X), \ ++ INSN_3(JMP, JSGE, X), \ ++ INSN_3(JMP, JSLE, X), \ ++ INSN_3(JMP, JSET, X), \ ++ /* Immediate based. */ \ ++ INSN_3(JMP, JEQ, K), \ ++ INSN_3(JMP, JNE, K), \ ++ INSN_3(JMP, JGT, K), \ ++ INSN_3(JMP, JLT, K), \ ++ INSN_3(JMP, JGE, K), \ ++ INSN_3(JMP, JLE, K), \ ++ INSN_3(JMP, JSGT, K), \ ++ INSN_3(JMP, JSLT, K), \ ++ INSN_3(JMP, JSGE, K), \ ++ INSN_3(JMP, JSLE, K), \ ++ INSN_3(JMP, JSET, K), \ ++ INSN_2(JMP, JA), \ ++ /* Store instructions. */ \ ++ /* Register based. */ \ ++ INSN_3(STX, MEM, B), \ ++ INSN_3(STX, MEM, H), \ ++ INSN_3(STX, MEM, W), \ ++ INSN_3(STX, MEM, DW), \ ++ INSN_3(STX, XADD, W), \ ++ INSN_3(STX, XADD, DW), \ ++ /* Immediate based. */ \ ++ INSN_3(ST, MEM, B), \ ++ INSN_3(ST, MEM, H), \ ++ INSN_3(ST, MEM, W), \ ++ INSN_3(ST, MEM, DW), \ ++ /* Load instructions. */ \ ++ /* Register based. */ \ ++ INSN_3(LDX, MEM, B), \ ++ INSN_3(LDX, MEM, H), \ ++ INSN_3(LDX, MEM, W), \ ++ INSN_3(LDX, MEM, DW), \ ++ /* Immediate based. */ \ ++ INSN_3(LD, IMM, DW) ++ ++bool bpf_opcode_in_insntable(u8 code) ++{ ++#define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true ++#define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true ++ static const bool public_insntable[256] = { ++ [0 ... 255] = false, ++ /* Now overwrite non-defaults ... */ ++ BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), ++ /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ ++ [BPF_LD | BPF_ABS | BPF_B] = true, ++ [BPF_LD | BPF_ABS | BPF_H] = true, ++ [BPF_LD | BPF_ABS | BPF_W] = true, ++ [BPF_LD | BPF_IND | BPF_B] = true, ++ [BPF_LD | BPF_IND | BPF_H] = true, ++ [BPF_LD | BPF_IND | BPF_W] = true, ++ }; ++#undef BPF_INSN_3_TBL ++#undef BPF_INSN_2_TBL ++ return public_insntable[code]; ++} ++ ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON + /** + * __bpf_prog_run - run eBPF program on a given context +- * @ctx: is the data we are operating on ++ * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers + * @insn: is the array of eBPF instructions ++ * @stack: is the eBPF storage stack + * + * Decode and execute eBPF instructions. + */ +-static unsigned int __bpf_prog_run(void *ctx, const struct bpf_insn *insn) ++static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) + { +- u64 stack[MAX_BPF_STACK / sizeof(u64)]; +- u64 regs[MAX_BPF_REG], tmp; +- static const void *jumptable[256] = { ++#define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y ++#define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z ++ static const void * const jumptable[256] = { + [0 ... 255] = &&default_label, + /* Now overwrite non-defaults ... */ +- /* 32 bit ALU operations */ +- [BPF_ALU | BPF_ADD | BPF_X] = &&ALU_ADD_X, +- [BPF_ALU | BPF_ADD | BPF_K] = &&ALU_ADD_K, +- [BPF_ALU | BPF_SUB | BPF_X] = &&ALU_SUB_X, +- [BPF_ALU | BPF_SUB | BPF_K] = &&ALU_SUB_K, +- [BPF_ALU | BPF_AND | BPF_X] = &&ALU_AND_X, +- [BPF_ALU | BPF_AND | BPF_K] = &&ALU_AND_K, +- [BPF_ALU | BPF_OR | BPF_X] = &&ALU_OR_X, +- [BPF_ALU | BPF_OR | BPF_K] = &&ALU_OR_K, +- [BPF_ALU | BPF_LSH | BPF_X] = &&ALU_LSH_X, +- [BPF_ALU | BPF_LSH | BPF_K] = &&ALU_LSH_K, +- [BPF_ALU | BPF_RSH | BPF_X] = &&ALU_RSH_X, +- [BPF_ALU | BPF_RSH | BPF_K] = &&ALU_RSH_K, +- [BPF_ALU | BPF_XOR | BPF_X] = &&ALU_XOR_X, +- [BPF_ALU | BPF_XOR | BPF_K] = &&ALU_XOR_K, +- [BPF_ALU | BPF_MUL | BPF_X] = &&ALU_MUL_X, +- [BPF_ALU | BPF_MUL | BPF_K] = &&ALU_MUL_K, +- [BPF_ALU | BPF_MOV | BPF_X] = &&ALU_MOV_X, +- [BPF_ALU | BPF_MOV | BPF_K] = &&ALU_MOV_K, +- [BPF_ALU | BPF_DIV | BPF_X] = &&ALU_DIV_X, +- [BPF_ALU | BPF_DIV | BPF_K] = &&ALU_DIV_K, +- [BPF_ALU | BPF_MOD | BPF_X] = &&ALU_MOD_X, +- [BPF_ALU | BPF_MOD | BPF_K] = &&ALU_MOD_K, +- [BPF_ALU | BPF_NEG] = &&ALU_NEG, +- [BPF_ALU | BPF_END | BPF_TO_BE] = &&ALU_END_TO_BE, +- [BPF_ALU | BPF_END | BPF_TO_LE] = &&ALU_END_TO_LE, +- /* 64 bit ALU operations */ +- [BPF_ALU64 | BPF_ADD | BPF_X] = &&ALU64_ADD_X, +- [BPF_ALU64 | BPF_ADD | BPF_K] = &&ALU64_ADD_K, +- [BPF_ALU64 | BPF_SUB | BPF_X] = &&ALU64_SUB_X, +- [BPF_ALU64 | BPF_SUB | BPF_K] = &&ALU64_SUB_K, +- [BPF_ALU64 | BPF_AND | BPF_X] = &&ALU64_AND_X, +- [BPF_ALU64 | BPF_AND | BPF_K] = &&ALU64_AND_K, +- [BPF_ALU64 | BPF_OR | BPF_X] = &&ALU64_OR_X, +- [BPF_ALU64 | BPF_OR | BPF_K] = &&ALU64_OR_K, +- [BPF_ALU64 | BPF_LSH | BPF_X] = &&ALU64_LSH_X, +- [BPF_ALU64 | BPF_LSH | BPF_K] = &&ALU64_LSH_K, +- [BPF_ALU64 | BPF_RSH | BPF_X] = &&ALU64_RSH_X, +- [BPF_ALU64 | BPF_RSH | BPF_K] = &&ALU64_RSH_K, +- [BPF_ALU64 | BPF_XOR | BPF_X] = &&ALU64_XOR_X, +- [BPF_ALU64 | BPF_XOR | BPF_K] = &&ALU64_XOR_K, +- [BPF_ALU64 | BPF_MUL | BPF_X] = &&ALU64_MUL_X, +- [BPF_ALU64 | BPF_MUL | BPF_K] = &&ALU64_MUL_K, +- [BPF_ALU64 | BPF_MOV | BPF_X] = &&ALU64_MOV_X, +- [BPF_ALU64 | BPF_MOV | BPF_K] = &&ALU64_MOV_K, +- [BPF_ALU64 | BPF_ARSH | BPF_X] = &&ALU64_ARSH_X, +- [BPF_ALU64 | BPF_ARSH | BPF_K] = &&ALU64_ARSH_K, +- [BPF_ALU64 | BPF_DIV | BPF_X] = &&ALU64_DIV_X, +- [BPF_ALU64 | BPF_DIV | BPF_K] = &&ALU64_DIV_K, +- [BPF_ALU64 | BPF_MOD | BPF_X] = &&ALU64_MOD_X, +- [BPF_ALU64 | BPF_MOD | BPF_K] = &&ALU64_MOD_K, +- [BPF_ALU64 | BPF_NEG] = &&ALU64_NEG, +- /* Call instruction */ +- [BPF_JMP | BPF_CALL] = &&JMP_CALL, +- [BPF_JMP | BPF_CALL | BPF_X] = &&JMP_TAIL_CALL, +- /* Jumps */ +- [BPF_JMP | BPF_JA] = &&JMP_JA, +- [BPF_JMP | BPF_JEQ | BPF_X] = &&JMP_JEQ_X, +- [BPF_JMP | BPF_JEQ | BPF_K] = &&JMP_JEQ_K, +- [BPF_JMP | BPF_JNE | BPF_X] = &&JMP_JNE_X, +- [BPF_JMP | BPF_JNE | BPF_K] = &&JMP_JNE_K, +- [BPF_JMP | BPF_JGT | BPF_X] = &&JMP_JGT_X, +- [BPF_JMP | BPF_JGT | BPF_K] = &&JMP_JGT_K, +- [BPF_JMP | BPF_JGE | BPF_X] = &&JMP_JGE_X, +- [BPF_JMP | BPF_JGE | BPF_K] = &&JMP_JGE_K, +- [BPF_JMP | BPF_JSGT | BPF_X] = &&JMP_JSGT_X, +- [BPF_JMP | BPF_JSGT | BPF_K] = &&JMP_JSGT_K, +- [BPF_JMP | BPF_JSGE | BPF_X] = &&JMP_JSGE_X, +- [BPF_JMP | BPF_JSGE | BPF_K] = &&JMP_JSGE_K, +- [BPF_JMP | BPF_JSET | BPF_X] = &&JMP_JSET_X, +- [BPF_JMP | BPF_JSET | BPF_K] = &&JMP_JSET_K, +- /* Program return */ +- [BPF_JMP | BPF_EXIT] = &&JMP_EXIT, +- /* Store instructions */ +- [BPF_STX | BPF_MEM | BPF_B] = &&STX_MEM_B, +- [BPF_STX | BPF_MEM | BPF_H] = &&STX_MEM_H, +- [BPF_STX | BPF_MEM | BPF_W] = &&STX_MEM_W, +- [BPF_STX | BPF_MEM | BPF_DW] = &&STX_MEM_DW, +- [BPF_STX | BPF_XADD | BPF_W] = &&STX_XADD_W, +- [BPF_STX | BPF_XADD | BPF_DW] = &&STX_XADD_DW, +- [BPF_ST | BPF_MEM | BPF_B] = &&ST_MEM_B, +- [BPF_ST | BPF_MEM | BPF_H] = &&ST_MEM_H, +- [BPF_ST | BPF_MEM | BPF_W] = &&ST_MEM_W, +- [BPF_ST | BPF_MEM | BPF_DW] = &&ST_MEM_DW, +- /* Load instructions */ +- [BPF_LDX | BPF_MEM | BPF_B] = &&LDX_MEM_B, +- [BPF_LDX | BPF_MEM | BPF_H] = &&LDX_MEM_H, +- [BPF_LDX | BPF_MEM | BPF_W] = &&LDX_MEM_W, +- [BPF_LDX | BPF_MEM | BPF_DW] = &&LDX_MEM_DW, +- [BPF_LD | BPF_ABS | BPF_W] = &&LD_ABS_W, +- [BPF_LD | BPF_ABS | BPF_H] = &&LD_ABS_H, +- [BPF_LD | BPF_ABS | BPF_B] = &&LD_ABS_B, +- [BPF_LD | BPF_IND | BPF_W] = &&LD_IND_W, +- [BPF_LD | BPF_IND | BPF_H] = &&LD_IND_H, +- [BPF_LD | BPF_IND | BPF_B] = &&LD_IND_B, +- [BPF_LD | BPF_IMM | BPF_DW] = &&LD_IMM_DW, ++ BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), ++ /* Non-UAPI available opcodes. */ ++ [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, ++ [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, + }; ++#undef BPF_INSN_3_LBL ++#undef BPF_INSN_2_LBL + u32 tail_call_cnt = 0; +- void *ptr; +- int off; + + #define CONT ({ insn++; goto select_insn; }) + #define CONT_JMP ({ insn++; goto select_insn; }) + +- FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; +- ARG1 = (u64) (unsigned long) ctx; +- +- /* Registers used in classic BPF programs need to be reset first. */ +- regs[BPF_REG_A] = 0; +- regs[BPF_REG_X] = 0; +- + select_insn: + goto *jumptable[insn->code]; + +- /* ALU */ +-#define ALU(OPCODE, OP) \ +- ALU64_##OPCODE##_X: \ +- DST = DST OP SRC; \ +- CONT; \ +- ALU_##OPCODE##_X: \ +- DST = (u32) DST OP (u32) SRC; \ +- CONT; \ +- ALU64_##OPCODE##_K: \ +- DST = DST OP IMM; \ +- CONT; \ +- ALU_##OPCODE##_K: \ +- DST = (u32) DST OP (u32) IMM; \ ++ /* Explicitly mask the register-based shift amounts with 63 or 31 ++ * to avoid undefined behavior. Normally this won't affect the ++ * generated code, for example, in case of native 64 bit archs such ++ * as x86-64 or arm64, the compiler is optimizing the AND away for ++ * the interpreter. In case of JITs, each of the JIT backends compiles ++ * the BPF shift operations to machine instructions which produce ++ * implementation-defined results in such a case; the resulting ++ * contents of the register may be arbitrary, but program behaviour ++ * as a whole remains defined. In other words, in case of JIT backends, ++ * the AND must /not/ be added to the emitted LSH/RSH/ARSH translation. ++ */ ++ /* ALU (shifts) */ ++#define SHT(OPCODE, OP) \ ++ ALU64_##OPCODE##_X: \ ++ DST = DST OP (SRC & 63); \ ++ CONT; \ ++ ALU_##OPCODE##_X: \ ++ DST = (u32) DST OP ((u32) SRC & 31); \ ++ CONT; \ ++ ALU64_##OPCODE##_K: \ ++ DST = DST OP IMM; \ ++ CONT; \ ++ ALU_##OPCODE##_K: \ ++ DST = (u32) DST OP (u32) IMM; \ ++ CONT; ++ /* ALU (rest) */ ++#define ALU(OPCODE, OP) \ ++ ALU64_##OPCODE##_X: \ ++ DST = DST OP SRC; \ ++ CONT; \ ++ ALU_##OPCODE##_X: \ ++ DST = (u32) DST OP (u32) SRC; \ ++ CONT; \ ++ ALU64_##OPCODE##_K: \ ++ DST = DST OP IMM; \ ++ CONT; \ ++ ALU_##OPCODE##_K: \ ++ DST = (u32) DST OP (u32) IMM; \ + CONT; +- + ALU(ADD, +) + ALU(SUB, -) + ALU(AND, &) + ALU(OR, |) +- ALU(LSH, <<) +- ALU(RSH, >>) + ALU(XOR, ^) + ALU(MUL, *) ++ SHT(LSH, <<) ++ SHT(RSH, >>) ++#undef SHT + #undef ALU + ALU_NEG: + DST = (u32) -DST; +@@ -359,51 +1392,49 @@ select_insn: + DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; + insn++; + CONT; ++ ALU_ARSH_X: ++ DST = (u64) (u32) (((s32) DST) >> (SRC & 31)); ++ CONT; ++ ALU_ARSH_K: ++ DST = (u64) (u32) (((s32) DST) >> IMM); ++ CONT; + ALU64_ARSH_X: +- (*(s64 *) &DST) >>= SRC; ++ (*(s64 *) &DST) >>= (SRC & 63); + CONT; + ALU64_ARSH_K: + (*(s64 *) &DST) >>= IMM; + CONT; + ALU64_MOD_X: +- if (unlikely(SRC == 0)) +- return 0; +- div64_u64_rem(DST, SRC, &tmp); +- DST = tmp; ++ div64_u64_rem(DST, SRC, &AX); ++ DST = AX; + CONT; + ALU_MOD_X: +- if (unlikely(SRC == 0)) +- return 0; +- tmp = (u32) DST; +- DST = do_div(tmp, (u32) SRC); ++ AX = (u32) DST; ++ DST = do_div(AX, (u32) SRC); + CONT; + ALU64_MOD_K: +- div64_u64_rem(DST, IMM, &tmp); +- DST = tmp; ++ div64_u64_rem(DST, IMM, &AX); ++ DST = AX; + CONT; + ALU_MOD_K: +- tmp = (u32) DST; +- DST = do_div(tmp, (u32) IMM); ++ AX = (u32) DST; ++ DST = do_div(AX, (u32) IMM); + CONT; + ALU64_DIV_X: +- if (unlikely(SRC == 0)) +- return 0; + DST = div64_u64(DST, SRC); + CONT; + ALU_DIV_X: +- if (unlikely(SRC == 0)) +- return 0; +- tmp = (u32) DST; +- do_div(tmp, (u32) SRC); +- DST = (u32) tmp; ++ AX = (u32) DST; ++ do_div(AX, (u32) SRC); ++ DST = (u32) AX; + CONT; + ALU64_DIV_K: + DST = div64_u64(DST, IMM); + CONT; + ALU_DIV_K: +- tmp = (u32) DST; +- do_div(tmp, (u32) IMM); +- DST = (u32) tmp; ++ AX = (u32) DST; ++ do_div(AX, (u32) IMM); ++ DST = (u32) AX; + CONT; + ALU_END_TO_BE: + switch (IMM) { +@@ -442,22 +1473,28 @@ select_insn: + BPF_R4, BPF_R5); + CONT; + ++ JMP_CALL_ARGS: ++ BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, ++ BPF_R3, BPF_R4, ++ BPF_R5, ++ insn + insn->off + 1); ++ CONT; ++ + JMP_TAIL_CALL: { + struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct bpf_prog *prog; +- u64 index = BPF_R3; ++ u32 index = BPF_R3; + + if (unlikely(index >= array->map.max_entries)) + goto out; +- + if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT)) + goto out; + + tail_call_cnt++; + + prog = READ_ONCE(array->ptrs[index]); +- if (unlikely(!prog)) ++ if (!prog) + goto out; + + /* ARG1 at this point is guaranteed to point to CTX from +@@ -470,97 +1507,49 @@ select_insn: + out: + CONT; + } +- /* JMP */ + JMP_JA: + insn += insn->off; + CONT; +- JMP_JEQ_X: +- if (DST == SRC) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JEQ_K: +- if (DST == IMM) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JNE_X: +- if (DST != SRC) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JNE_K: +- if (DST != IMM) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JGT_X: +- if (DST > SRC) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JGT_K: +- if (DST > IMM) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JGE_X: +- if (DST >= SRC) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JGE_K: +- if (DST >= IMM) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSGT_X: +- if (((s64) DST) > ((s64) SRC)) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSGT_K: +- if (((s64) DST) > ((s64) IMM)) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSGE_X: +- if (((s64) DST) >= ((s64) SRC)) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSGE_K: +- if (((s64) DST) >= ((s64) IMM)) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSET_X: +- if (DST & SRC) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; +- JMP_JSET_K: +- if (DST & IMM) { +- insn += insn->off; +- CONT_JMP; +- } +- CONT; + JMP_EXIT: + return BPF_R0; +- ++ /* JMP */ ++#define COND_JMP(SIGN, OPCODE, CMP_OP) \ ++ JMP_##OPCODE##_X: \ ++ if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ ++ insn += insn->off; \ ++ CONT_JMP; \ ++ } \ ++ CONT; \ ++ JMP32_##OPCODE##_X: \ ++ if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ ++ insn += insn->off; \ ++ CONT_JMP; \ ++ } \ ++ CONT; \ ++ JMP_##OPCODE##_K: \ ++ if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ ++ insn += insn->off; \ ++ CONT_JMP; \ ++ } \ ++ CONT; \ ++ JMP32_##OPCODE##_K: \ ++ if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ ++ insn += insn->off; \ ++ CONT_JMP; \ ++ } \ ++ CONT; ++ COND_JMP(u, JEQ, ==) ++ COND_JMP(u, JNE, !=) ++ COND_JMP(u, JGT, >) ++ COND_JMP(u, JLT, <) ++ COND_JMP(u, JGE, >=) ++ COND_JMP(u, JLE, <=) ++ COND_JMP(u, JSET, &) ++ COND_JMP(s, JSGT, >) ++ COND_JMP(s, JSLT, <) ++ COND_JMP(s, JSGE, >=) ++ COND_JMP(s, JSLE, <=) ++#undef COND_JMP + /* STX and ST and LDX*/ + #define LDST(SIZEOP, SIZE) \ + STX_MEM_##SIZEOP: \ +@@ -586,77 +1575,108 @@ out: + atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) + (DST + insn->off)); + CONT; +- LD_ABS_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */ +- off = IMM; +-load_word: +- /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are +- * only appearing in the programs where ctx == +- * skb. All programs keep 'ctx' in regs[BPF_REG_CTX] +- * == BPF_R6, bpf_convert_filter() saves it in BPF_R6, +- * internal BPF verifier will check that BPF_R6 == +- * ctx. +- * +- * BPF_ABS and BPF_IND are wrappers of function calls, +- * so they scratch BPF_R1-BPF_R5 registers, preserve +- * BPF_R6-BPF_R9, and store return value into BPF_R0. +- * +- * Implicit input: +- * ctx == skb == BPF_R6 == CTX +- * +- * Explicit input: +- * SRC == any register +- * IMM == 32-bit immediate ++ ++ default_label: ++ /* If we ever reach this, we have a bug somewhere. Die hard here ++ * instead of just returning 0; we could be somewhere in a subprog, ++ * so execution could continue otherwise which we do /not/ want. + * +- * Output: +- * BPF_R0 - 8/16/32-bit skb data converted to cpu endianness ++ * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). + */ ++ pr_warn("BPF interpreter: unknown opcode %02x\n", insn->code); ++ BUG_ON(1); ++ return 0; ++} + +- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 4, &tmp); +- if (likely(ptr != NULL)) { +- BPF_R0 = get_unaligned_be32(ptr); +- CONT; +- } ++#define PROG_NAME(stack_size) __bpf_prog_run##stack_size ++#define DEFINE_BPF_PROG_RUN(stack_size) \ ++static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ ++{ \ ++ u64 stack[stack_size / sizeof(u64)]; \ ++ u64 regs[MAX_BPF_EXT_REG]; \ ++\ ++ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ ++ ARG1 = (u64) (unsigned long) ctx; \ ++ return ___bpf_prog_run(regs, insn, stack); \ ++} + +- return 0; +- LD_ABS_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + imm32)) */ +- off = IMM; +-load_half: +- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 2, &tmp); +- if (likely(ptr != NULL)) { +- BPF_R0 = get_unaligned_be16(ptr); +- CONT; +- } ++#define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size ++#define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ ++static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ ++ const struct bpf_insn *insn) \ ++{ \ ++ u64 stack[stack_size / sizeof(u64)]; \ ++ u64 regs[MAX_BPF_EXT_REG]; \ ++\ ++ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ ++ BPF_R1 = r1; \ ++ BPF_R2 = r2; \ ++ BPF_R3 = r3; \ ++ BPF_R4 = r4; \ ++ BPF_R5 = r5; \ ++ return ___bpf_prog_run(regs, insn, stack); \ ++} + +- return 0; +- LD_ABS_B: /* BPF_R0 = *(u8 *) (skb->data + imm32) */ +- off = IMM; +-load_byte: +- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 1, &tmp); +- if (likely(ptr != NULL)) { +- BPF_R0 = *(u8 *)ptr; +- CONT; +- } ++#define EVAL1(FN, X) FN(X) ++#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) ++#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) ++#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) ++#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) ++#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) ++ ++EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); ++EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); ++EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); ++ ++EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); ++EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); ++EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); ++ ++#define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), ++ ++static unsigned int (*interpreters[])(const void *ctx, ++ const struct bpf_insn *insn) = { ++EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) ++EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) ++EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) ++}; ++#undef PROG_NAME_LIST ++#define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), ++static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, ++ const struct bpf_insn *insn) = { ++EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) ++EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) ++EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) ++}; ++#undef PROG_NAME_LIST + +- return 0; +- LD_IND_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + src_reg + imm32)) */ +- off = IMM + SRC; +- goto load_word; +- LD_IND_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + src_reg + imm32)) */ +- off = IMM + SRC; +- goto load_half; +- LD_IND_B: /* BPF_R0 = *(u8 *) (skb->data + src_reg + imm32) */ +- off = IMM + SRC; +- goto load_byte; ++void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) ++{ ++ stack_depth = max_t(u32, stack_depth, 1); ++ insn->off = (s16) insn->imm; ++ insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - ++ __bpf_call_base_args; ++ insn->code = BPF_JMP | BPF_CALL_ARGS; ++} + +- default_label: +- /* If we ever reach this, we have a bug somewhere. */ +- WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code); +- return 0; ++#else ++static unsigned int __bpf_prog_ret0_warn(const void *ctx, ++ const struct bpf_insn *insn) ++{ ++ /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON ++ * is not working properly, so warn about it! ++ */ ++ WARN_ON_ONCE(1); ++ return 0; + } ++#endif + + bool bpf_prog_array_compatible(struct bpf_array *array, + const struct bpf_prog *fp) + { ++ if (fp->kprobe_override) ++ return false; ++ + if (!array->owner_prog_type) { + /* There's no owner yet where we could check for + * compatibility. +@@ -691,18 +1711,62 @@ static int bpf_check_tail_call(const str + return 0; + } + ++static void bpf_prog_select_func(struct bpf_prog *fp) ++{ ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++ u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); ++ ++ fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; ++#else ++ fp->bpf_func = __bpf_prog_ret0_warn; ++#endif ++} ++ + /** + * bpf_prog_select_runtime - select exec runtime for BPF program + * @fp: bpf_prog populated with internal BPF program ++ * @err: pointer to error variable + * + * Try to JIT eBPF program, if JIT is not available, use interpreter. + * The BPF program will be executed via BPF_PROG_RUN() macro. + */ +-int bpf_prog_select_runtime(struct bpf_prog *fp) ++struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) + { +- fp->bpf_func = (void *) __bpf_prog_run; ++ /* In case of BPF to BPF calls, verifier did all the prep ++ * work with regards to JITing, etc. ++ */ ++ if (fp->bpf_func) ++ goto finalize; ++ ++ bpf_prog_select_func(fp); ++ ++ /* eBPF JITs can rewrite the program in case constant ++ * blinding is active. However, in case of error during ++ * blinding, bpf_int_jit_compile() must always return a ++ * valid program, which in this case would simply not ++ * be JITed, but falls back to the interpreter. ++ */ ++ if (!bpf_prog_is_dev_bound(fp->aux)) { ++ *err = bpf_prog_alloc_jited_linfo(fp); ++ if (*err) ++ return fp; ++ ++ fp = bpf_int_jit_compile(fp); ++ if (!fp->jited) { ++ bpf_prog_free_jited_linfo(fp); ++#ifdef CONFIG_BPF_JIT_ALWAYS_ON ++ *err = -ENOTSUPP; ++ return fp; ++#endif ++ } else { ++ bpf_prog_free_unused_jited_linfo(fp); ++ } ++ } else { ++ *err = -EINVAL; ++ return fp; ++ } + +- bpf_int_jit_compile(fp); ++finalize: + bpf_prog_lock_ro(fp); + + /* The tail call compatibility check can only be done at +@@ -710,16 +1774,238 @@ int bpf_prog_select_runtime(struct bpf_p + * with JITed or non JITed program concatenations and not + * all eBPF JITs might immediately support all features. + */ +- return bpf_check_tail_call(fp); ++ *err = bpf_check_tail_call(fp); ++ ++ return fp; + } + EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); + ++static unsigned int __bpf_prog_ret1(const void *ctx, ++ const struct bpf_insn *insn) ++{ ++ return 1; ++} ++ ++static struct bpf_prog_dummy { ++ struct bpf_prog prog; ++} dummy_bpf_prog = { ++ .prog = { ++ .bpf_func = __bpf_prog_ret1, ++ }, ++}; ++ ++/* to avoid allocating empty bpf_prog_array for cgroups that ++ * don't have bpf program attached use one global 'empty_prog_array' ++ * It will not be modified the caller of bpf_prog_array_alloc() ++ * (since caller requested prog_cnt == 0) ++ * that pointer should be 'freed' by bpf_prog_array_free() ++ */ ++static struct { ++ struct bpf_prog_array hdr; ++ struct bpf_prog *null_prog; ++} empty_prog_array = { ++ .null_prog = NULL, ++}; ++ ++struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) ++{ ++ if (prog_cnt) ++ return kzalloc(sizeof(struct bpf_prog_array) + ++ sizeof(struct bpf_prog_array_item) * ++ (prog_cnt + 1), ++ flags); ++ ++ return &empty_prog_array.hdr; ++} ++ ++void bpf_prog_array_free(struct bpf_prog_array *progs) ++{ ++ if (!progs || progs == &empty_prog_array.hdr) ++ return; ++ kfree_rcu(progs, rcu); ++} ++ ++int bpf_prog_array_length(struct bpf_prog_array *array) ++{ ++ struct bpf_prog_array_item *item; ++ u32 cnt = 0; ++ ++ for (item = array->items; item->prog; item++) ++ if (item->prog != &dummy_bpf_prog.prog) ++ cnt++; ++ return cnt; ++} ++ ++bool bpf_prog_array_is_empty(struct bpf_prog_array *array) ++{ ++ struct bpf_prog_array_item *item; ++ ++ for (item = array->items; item->prog; item++) ++ if (item->prog != &dummy_bpf_prog.prog) ++ return false; ++ return true; ++} ++ ++static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, ++ u32 *prog_ids, ++ u32 request_cnt) ++{ ++ struct bpf_prog_array_item *item; ++ int i = 0; ++ ++ for (item = array->items; item->prog; item++) { ++ if (item->prog == &dummy_bpf_prog.prog) ++ continue; ++ prog_ids[i] = item->prog->aux->id; ++ if (++i == request_cnt) { ++ item++; ++ break; ++ } ++ } ++ ++ return !!(item->prog); ++} ++ ++int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, ++ __u32 __user *prog_ids, u32 cnt) ++{ ++ unsigned long err = 0; ++ bool nospc; ++ u32 *ids; ++ ++ /* users of this function are doing: ++ * cnt = bpf_prog_array_length(); ++ * if (cnt > 0) ++ * bpf_prog_array_copy_to_user(..., cnt); ++ * so below kcalloc doesn't need extra cnt > 0 check. ++ */ ++ ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); ++ if (!ids) ++ return -ENOMEM; ++ nospc = bpf_prog_array_copy_core(array, ids, cnt); ++ err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); ++ kfree(ids); ++ if (err) ++ return -EFAULT; ++ if (nospc) ++ return -ENOSPC; ++ return 0; ++} ++ ++void bpf_prog_array_delete_safe(struct bpf_prog_array *array, ++ struct bpf_prog *old_prog) ++{ ++ struct bpf_prog_array_item *item; ++ ++ for (item = array->items; item->prog; item++) ++ if (item->prog == old_prog) { ++ WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); ++ break; ++ } ++} ++ ++int bpf_prog_array_copy(struct bpf_prog_array *old_array, ++ struct bpf_prog *exclude_prog, ++ struct bpf_prog *include_prog, ++ struct bpf_prog_array **new_array) ++{ ++ int new_prog_cnt, carry_prog_cnt = 0; ++ struct bpf_prog_array_item *existing; ++ struct bpf_prog_array *array; ++ bool found_exclude = false; ++ int new_prog_idx = 0; ++ ++ /* Figure out how many existing progs we need to carry over to ++ * the new array. ++ */ ++ if (old_array) { ++ existing = old_array->items; ++ for (; existing->prog; existing++) { ++ if (existing->prog == exclude_prog) { ++ found_exclude = true; ++ continue; ++ } ++ if (existing->prog != &dummy_bpf_prog.prog) ++ carry_prog_cnt++; ++ if (existing->prog == include_prog) ++ return -EEXIST; ++ } ++ } ++ ++ if (exclude_prog && !found_exclude) ++ return -ENOENT; ++ ++ /* How many progs (not NULL) will be in the new array? */ ++ new_prog_cnt = carry_prog_cnt; ++ if (include_prog) ++ new_prog_cnt += 1; ++ ++ /* Do we have any prog (not NULL) in the new array? */ ++ if (!new_prog_cnt) { ++ *new_array = NULL; ++ return 0; ++ } ++ ++ /* +1 as the end of prog_array is marked with NULL */ ++ array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); ++ if (!array) ++ return -ENOMEM; ++ ++ /* Fill in the new prog array */ ++ if (carry_prog_cnt) { ++ existing = old_array->items; ++ for (; existing->prog; existing++) ++ if (existing->prog != exclude_prog && ++ existing->prog != &dummy_bpf_prog.prog) { ++ array->items[new_prog_idx++].prog = ++ existing->prog; ++ } ++ } ++ if (include_prog) ++ array->items[new_prog_idx++].prog = include_prog; ++ array->items[new_prog_idx].prog = NULL; ++ *new_array = array; ++ return 0; ++} ++ ++int bpf_prog_array_copy_info(struct bpf_prog_array *array, ++ u32 *prog_ids, u32 request_cnt, ++ u32 *prog_cnt) ++{ ++ u32 cnt = 0; ++ ++ if (array) ++ cnt = bpf_prog_array_length(array); ++ ++ *prog_cnt = cnt; ++ ++ /* return early if user requested only program count or nothing to copy */ ++ if (!request_cnt || !cnt) ++ return 0; ++ ++ /* this function is called under trace/bpf_trace.c: bpf_event_mutex */ ++ return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC ++ : 0; ++} ++ + static void bpf_prog_free_deferred(struct work_struct *work) + { + struct bpf_prog_aux *aux; ++ int i; + + aux = container_of(work, struct bpf_prog_aux, work); +- bpf_jit_free(aux->prog); ++#ifdef CONFIG_PERF_EVENTS ++ if (aux->prog->has_callchain_buf) ++ put_callchain_buffers(); ++#endif ++ for (i = 0; i < aux->func_cnt; i++) ++ bpf_jit_free(aux->func[i]); ++ if (aux->func_cnt) { ++ kfree(aux->func); ++ bpf_prog_unlock_free(aux->prog); ++ } else { ++ bpf_jit_free(aux->prog); ++ } + } + + /* Free internal BPF program */ +@@ -740,7 +2026,7 @@ void bpf_user_rnd_init_once(void) + prandom_init_once(&bpf_user_rnd_state); + } + +-u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_0(bpf_user_rnd_u32) + { + /* Should someone ever have the rather unwise idea to use some + * of the registers passed into this function, then note that +@@ -753,7 +2039,7 @@ u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 + + state = &get_cpu_var(bpf_user_rnd_state); + res = prandom_u32_state(state); +- put_cpu_var(state); ++ put_cpu_var(bpf_user_rnd_state); + + return res; + } +@@ -762,18 +2048,36 @@ u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 + const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; + const struct bpf_func_proto bpf_map_update_elem_proto __weak; + const struct bpf_func_proto bpf_map_delete_elem_proto __weak; ++const struct bpf_func_proto bpf_map_push_elem_proto __weak; ++const struct bpf_func_proto bpf_map_pop_elem_proto __weak; ++const struct bpf_func_proto bpf_map_peek_elem_proto __weak; ++const struct bpf_func_proto bpf_spin_lock_proto __weak; ++const struct bpf_func_proto bpf_spin_unlock_proto __weak; + + const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; + const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; ++const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; + const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; ++ + const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; + const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; + const struct bpf_func_proto bpf_get_current_comm_proto __weak; ++const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; ++const struct bpf_func_proto bpf_get_local_storage_proto __weak; ++ + const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) + { + return NULL; + } + ++u64 __weak ++bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, ++ void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) ++{ ++ return -ENOTSUPP; ++} ++EXPORT_SYMBOL_GPL(bpf_event_output); ++ + /* Always built-in helper functions. */ + const struct bpf_func_proto bpf_tail_call_proto = { + .func = NULL, +@@ -784,9 +2088,34 @@ const struct bpf_func_proto bpf_tail_cal + .arg3_type = ARG_ANYTHING, + }; + +-/* For classic BPF JITs that don't implement bpf_int_jit_compile(). */ +-void __weak bpf_int_jit_compile(struct bpf_prog *prog) ++/* Stub for JITs that only support cBPF. eBPF programs are interpreted. ++ * It is encouraged to implement bpf_int_jit_compile() instead, so that ++ * eBPF and implicitly also cBPF can get JITed! ++ */ ++struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) + { ++ return prog; ++} ++ ++/* Stub for JITs that support eBPF. All cBPF code gets transformed into ++ * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). ++ */ ++void __weak bpf_jit_compile(struct bpf_prog *prog) ++{ ++} ++ ++bool __weak bpf_helper_changes_pkt_data(void *func) ++{ ++ return false; ++} ++ ++/* Return TRUE if the JIT backend wants verifier to enable sub-register usage ++ * analysis code and wants explicit zero extension inserted by verifier. ++ * Otherwise, return FALSE. ++ */ ++bool __weak bpf_jit_needs_zext(void) ++{ ++ return false; + } + + /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call +@@ -797,3 +2126,13 @@ int __weak skb_copy_bits(const struct sk + { + return -EFAULT; + } ++ ++DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); ++EXPORT_SYMBOL(bpf_stats_enabled_key); ++ ++/* All definitions of tracepoints related to BPF. */ ++#define CREATE_TRACE_POINTS ++#include ++ ++EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); ++EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); +--- /dev/null ++++ b/kernel/bpf/devmap.c +@@ -0,0 +1,698 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io ++ */ ++ ++/* Devmaps primary use is as a backend map for XDP BPF helper call ++ * bpf_redirect_map(). Because XDP is mostly concerned with performance we ++ * spent some effort to ensure the datapath with redirect maps does not use ++ * any locking. This is a quick note on the details. ++ * ++ * We have three possible paths to get into the devmap control plane bpf ++ * syscalls, bpf programs, and driver side xmit/flush operations. A bpf syscall ++ * will invoke an update, delete, or lookup operation. To ensure updates and ++ * deletes appear atomic from the datapath side xchg() is used to modify the ++ * netdev_map array. Then because the datapath does a lookup into the netdev_map ++ * array (read-only) from an RCU critical section we use call_rcu() to wait for ++ * an rcu grace period before free'ing the old data structures. This ensures the ++ * datapath always has a valid copy. However, the datapath does a "flush" ++ * operation that pushes any pending packets in the driver outside the RCU ++ * critical section. Each bpf_dtab_netdev tracks these pending operations using ++ * a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed until ++ * this list is empty, indicating outstanding flush operations have completed. ++ * ++ * BPF syscalls may race with BPF program calls on any of the update, delete ++ * or lookup operations. As noted above the xchg() operation also keep the ++ * netdev_map consistent in this case. From the devmap side BPF programs ++ * calling into these operations are the same as multiple user space threads ++ * making system calls. ++ * ++ * Finally, any of the above may race with a netdev_unregister notifier. The ++ * unregister notifier must search for net devices in the map structure that ++ * contain a reference to the net device and remove them. This is a two step ++ * process (a) dereference the bpf_dtab_netdev object in netdev_map and (b) ++ * check to see if the ifindex is the same as the net_device being removed. ++ * When removing the dev a cmpxchg() is used to ensure the correct dev is ++ * removed, in the case of a concurrent update or delete operation it is ++ * possible that the initially referenced dev is no longer in the map. As the ++ * notifier hook walks the map we know that new dev references can not be ++ * added by the user because core infrastructure ensures dev_get_by_index() ++ * calls will fail at this point. ++ * ++ * The devmap_hash type is a map type which interprets keys as ifindexes and ++ * indexes these using a hashmap. This allows maps that use ifindex as key to be ++ * densely packed instead of having holes in the lookup array for unused ++ * ifindexes. The setup and packet enqueue/send code is shared between the two ++ * types of devmap; only the lookup and insertion is different. ++ */ ++#include ++#include ++#include ++#include ++ ++#define DEV_CREATE_FLAG_MASK \ ++ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) ++ ++#define DEV_MAP_BULK_SIZE 16 ++struct bpf_dtab_netdev; ++ ++struct xdp_bulk_queue { ++ struct xdp_frame *q[DEV_MAP_BULK_SIZE]; ++ struct list_head flush_node; ++ struct net_device *dev_rx; ++ struct bpf_dtab_netdev *obj; ++ unsigned int count; ++}; ++ ++struct bpf_dtab_netdev { ++ struct net_device *dev; /* must be first member, due to tracepoint */ ++ struct hlist_node index_hlist; ++ struct bpf_dtab *dtab; ++ struct xdp_bulk_queue __percpu *bulkq; ++ struct rcu_head rcu; ++ unsigned int idx; /* keep track of map index for tracepoint */ ++}; ++ ++struct bpf_dtab { ++ struct bpf_map map; ++ struct bpf_dtab_netdev **netdev_map; /* DEVMAP type only */ ++ struct list_head __percpu *flush_list; ++ struct list_head list; ++ ++ /* these are only used for DEVMAP_HASH type maps */ ++ struct hlist_head *dev_index_head; ++ spinlock_t index_lock; ++ unsigned int items; ++ u32 n_buckets; ++}; ++ ++static DEFINE_SPINLOCK(dev_map_lock); ++static LIST_HEAD(dev_map_list); ++ ++static struct hlist_head *dev_map_create_hash(unsigned int entries, ++ int numa_node) ++{ ++ int i; ++ struct hlist_head *hash; ++ ++ hash = bpf_map_area_alloc(entries * sizeof(*hash), numa_node); ++ if (hash != NULL) ++ for (i = 0; i < entries; i++) ++ INIT_HLIST_HEAD(&hash[i]); ++ ++ return hash; ++} ++ ++static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab, ++ int idx) ++{ ++ return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)]; ++} ++ ++static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr) ++{ ++ int err, cpu; ++ u64 cost; ++ ++ /* check sanity of attributes */ ++ if (attr->max_entries == 0 || attr->key_size != 4 || ++ attr->value_size != 4 || attr->map_flags & ~DEV_CREATE_FLAG_MASK) ++ return -EINVAL; ++ ++ /* Lookup returns a pointer straight to dev->ifindex, so make sure the ++ * verifier prevents writes from the BPF side ++ */ ++ attr->map_flags |= BPF_F_RDONLY_PROG; ++ ++ ++ bpf_map_init_from_attr(&dtab->map, attr); ++ ++ /* make sure page count doesn't overflow */ ++ cost = (u64) sizeof(struct list_head) * num_possible_cpus(); ++ ++ if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { ++ dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries); ++ ++ if (!dtab->n_buckets) /* Overflow check */ ++ return -EINVAL; ++ cost += (u64) sizeof(struct hlist_head) * dtab->n_buckets; ++ } else { ++ cost += (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *); ++ } ++ ++ /* if map size is larger than memlock limit, reject it */ ++ err = bpf_map_charge_init(&dtab->map.memory, cost); ++ if (err) ++ return -EINVAL; ++ ++ dtab->flush_list = alloc_percpu(struct list_head); ++ if (!dtab->flush_list) ++ goto free_charge; ++ ++ for_each_possible_cpu(cpu) ++ INIT_LIST_HEAD(per_cpu_ptr(dtab->flush_list, cpu)); ++ ++ if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { ++ dtab->dev_index_head = dev_map_create_hash(dtab->n_buckets, ++ dtab->map.numa_node); ++ if (!dtab->dev_index_head) ++ goto free_percpu; ++ ++ spin_lock_init(&dtab->index_lock); ++ } else { ++ dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries * ++ sizeof(struct bpf_dtab_netdev *), ++ dtab->map.numa_node); ++ if (!dtab->netdev_map) ++ goto free_percpu; ++ } ++ ++ return 0; ++ ++free_percpu: ++ free_percpu(dtab->flush_list); ++free_charge: ++ bpf_map_charge_finish(&dtab->map.memory); ++ return -ENOMEM; ++} ++ ++static struct bpf_map *dev_map_alloc(union bpf_attr *attr) ++{ ++ struct bpf_dtab *dtab; ++ int err; ++ ++ if (!capable(CAP_NET_ADMIN)) ++ return ERR_PTR(-EPERM); ++ ++ dtab = kzalloc(sizeof(*dtab), GFP_USER); ++ if (!dtab) ++ return ERR_PTR(-ENOMEM); ++ ++ err = dev_map_init_map(dtab, attr); ++ if (err) { ++ kfree(dtab); ++ return ERR_PTR(err); ++ } ++ ++ spin_lock(&dev_map_lock); ++ list_add_tail_rcu(&dtab->list, &dev_map_list); ++ spin_unlock(&dev_map_lock); ++ ++ return &dtab->map; ++} ++ ++static void dev_map_free(struct bpf_map *map) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ int i, cpu; ++ ++ /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, ++ * so the programs (can be more than one that used this map) were ++ * disconnected from events. Wait for outstanding critical sections in ++ * these programs to complete. The rcu critical section only guarantees ++ * no further reads against netdev_map. It does __not__ ensure pending ++ * flush operations (if any) are complete. ++ */ ++ ++ spin_lock(&dev_map_lock); ++ list_del_rcu(&dtab->list); ++ spin_unlock(&dev_map_lock); ++ ++ bpf_clear_redirect_map(map); ++ synchronize_rcu(); ++ ++ /* Make sure prior __dev_map_entry_free() have completed. */ ++ rcu_barrier(); ++ ++ /* To ensure all pending flush operations have completed wait for flush ++ * list to empty on _all_ cpus. ++ * Because the above synchronize_rcu() ensures the map is disconnected ++ * from the program we can assume no new items will be added. ++ */ ++ for_each_online_cpu(cpu) { ++ struct list_head *flush_list = per_cpu_ptr(dtab->flush_list, cpu); ++ ++ while (!list_empty(flush_list)) ++ cond_resched(); ++ } ++ ++ if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { ++ for (i = 0; i < dtab->n_buckets; i++) { ++ struct bpf_dtab_netdev *dev; ++ struct hlist_head *head; ++ struct hlist_node *next; ++ ++ head = dev_map_index_hash(dtab, i); ++ ++ hlist_for_each_entry_safe(dev, next, head, index_hlist) { ++ hlist_del_rcu(&dev->index_hlist); ++ free_percpu(dev->bulkq); ++ dev_put(dev->dev); ++ kfree(dev); ++ } ++ } ++ ++ bpf_map_area_free(dtab->dev_index_head); ++ } else { ++ for (i = 0; i < dtab->map.max_entries; i++) { ++ struct bpf_dtab_netdev *dev; ++ ++ dev = dtab->netdev_map[i]; ++ if (!dev) ++ continue; ++ ++ free_percpu(dev->bulkq); ++ dev_put(dev->dev); ++ kfree(dev); ++ } ++ ++ bpf_map_area_free(dtab->netdev_map); ++ } ++ ++ free_percpu(dtab->flush_list); ++ kfree(dtab); ++} ++ ++static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ u32 index = key ? *(u32 *)key : U32_MAX; ++ u32 *next = next_key; ++ ++ if (index >= dtab->map.max_entries) { ++ *next = 0; ++ return 0; ++ } ++ ++ if (index == dtab->map.max_entries - 1) ++ return -ENOENT; ++ *next = index + 1; ++ return 0; ++} ++ ++struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct hlist_head *head = dev_map_index_hash(dtab, key); ++ struct bpf_dtab_netdev *dev; ++ ++ hlist_for_each_entry_rcu(dev, head, index_hlist) ++ if (dev->idx == key) ++ return dev; ++ ++ return NULL; ++} ++ ++static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, ++ void *next_key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ u32 idx, *next = next_key; ++ struct bpf_dtab_netdev *dev, *next_dev; ++ struct hlist_head *head; ++ int i = 0; ++ ++ if (!key) ++ goto find_first; ++ ++ idx = *(u32 *)key; ++ ++ dev = __dev_map_hash_lookup_elem(map, idx); ++ if (!dev) ++ goto find_first; ++ ++ next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&dev->index_hlist)), ++ struct bpf_dtab_netdev, index_hlist); ++ ++ if (next_dev) { ++ *next = next_dev->idx; ++ return 0; ++ } ++ ++ i = idx & (dtab->n_buckets - 1); ++ i++; ++ ++ find_first: ++ for (; i < dtab->n_buckets; i++) { ++ head = dev_map_index_hash(dtab, i); ++ ++ next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)), ++ struct bpf_dtab_netdev, ++ index_hlist); ++ if (next_dev) { ++ *next = next_dev->idx; ++ return 0; ++ } ++ } ++ ++ return -ENOENT; ++} ++ ++/* __dev_map_flush is called from xdp_do_flush_map() which _must_ be signaled ++ * from the driver before returning from its napi->poll() routine. The poll() ++ * routine is called either from busy_poll context or net_rx_action signaled ++ * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the ++ * net device can be torn down. On devmap tear down we ensure the flush list ++ * is empty before completing to ensure all flush operations have completed. ++ */ ++void __dev_map_flush(struct bpf_map *map) ++{ ++} ++ ++/* rcu_read_lock (from syscall and BPF contexts) ensures that if a delete and/or ++ * update happens in parallel here a dev_put wont happen until after reading the ++ * ifindex. ++ */ ++struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct bpf_dtab_netdev *obj; ++ ++ if (key >= map->max_entries) ++ return NULL; ++ ++ obj = READ_ONCE(dtab->netdev_map[key]); ++ return obj; ++} ++ ++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, ++ struct net_device *dev_rx) ++{ ++ return -EOPNOTSUPP; ++} ++ ++int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, ++ struct bpf_prog *xdp_prog) ++{ ++ return -EOPNOTSUPP; ++} ++ ++static void *dev_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key); ++ struct net_device *dev = obj ? obj->dev : NULL; ++ ++ return dev ? &dev->ifindex : NULL; ++} ++ ++static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map, ++ *(u32 *)key); ++ struct net_device *dev = obj ? obj->dev : NULL; ++ ++ return dev ? &dev->ifindex : NULL; ++} ++ ++static void __dev_map_entry_free(struct rcu_head *rcu) ++{ ++ struct bpf_dtab_netdev *dev; ++ ++ dev = container_of(rcu, struct bpf_dtab_netdev, rcu); ++ free_percpu(dev->bulkq); ++ dev_put(dev->dev); ++ kfree(dev); ++} ++ ++static int dev_map_delete_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct bpf_dtab_netdev *old_dev; ++ int k = *(u32 *)key; ++ ++ if (k >= map->max_entries) ++ return -EINVAL; ++ ++ /* Use call_rcu() here to ensure any rcu critical sections have ++ * completed, but this does not guarantee a flush has happened ++ * yet. Because driver side rcu_read_lock/unlock only protects the ++ * running XDP program. However, for pending flush operations the ++ * dev and ctx are stored in another per cpu map. And additionally, ++ * the driver tear down ensures all soft irqs are complete before ++ * removing the net device in the case of dev_put equals zero. ++ */ ++ old_dev = xchg(&dtab->netdev_map[k], NULL); ++ if (old_dev) ++ call_rcu(&old_dev->rcu, __dev_map_entry_free); ++ return 0; ++} ++ ++static int dev_map_hash_delete_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct bpf_dtab_netdev *old_dev; ++ int k = *(u32 *)key; ++ unsigned long flags; ++ int ret = -ENOENT; ++ ++ spin_lock_irqsave(&dtab->index_lock, flags); ++ ++ old_dev = __dev_map_hash_lookup_elem(map, k); ++ if (old_dev) { ++ dtab->items--; ++ hlist_del_init_rcu(&old_dev->index_hlist); ++ call_rcu(&old_dev->rcu, __dev_map_entry_free); ++ ret = 0; ++ } ++ spin_unlock_irqrestore(&dtab->index_lock, flags); ++ ++ return ret; ++} ++ ++static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, ++ struct bpf_dtab *dtab, ++ u32 ifindex, ++ unsigned int idx) ++{ ++ gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN; ++ struct bpf_dtab_netdev *dev; ++ struct xdp_bulk_queue *bq; ++ int cpu; ++ ++ dev = kmalloc_node(sizeof(*dev), gfp, dtab->map.numa_node); ++ if (!dev) ++ return ERR_PTR(-ENOMEM); ++ ++ dev->bulkq = __alloc_percpu_gfp(sizeof(*dev->bulkq), ++ sizeof(void *), gfp); ++ if (!dev->bulkq) { ++ kfree(dev); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ for_each_possible_cpu(cpu) { ++ bq = per_cpu_ptr(dev->bulkq, cpu); ++ bq->obj = dev; ++ } ++ ++ dev->dev = dev_get_by_index(net, ifindex); ++ if (!dev->dev) { ++ free_percpu(dev->bulkq); ++ kfree(dev); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ dev->idx = idx; ++ dev->dtab = dtab; ++ ++ return dev; ++} ++ ++static int __dev_map_update_elem(struct net *net, struct bpf_map *map, ++ void *key, void *value, u64 map_flags) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct bpf_dtab_netdev *dev, *old_dev; ++ u32 ifindex = *(u32 *)value; ++ u32 i = *(u32 *)key; ++ ++ if (unlikely(map_flags > BPF_EXIST)) ++ return -EINVAL; ++ if (unlikely(i >= dtab->map.max_entries)) ++ return -E2BIG; ++ if (unlikely(map_flags == BPF_NOEXIST)) ++ return -EEXIST; ++ ++ if (!ifindex) { ++ dev = NULL; ++ } else { ++ dev = __dev_map_alloc_node(net, dtab, ifindex, i); ++ if (IS_ERR(dev)) ++ return PTR_ERR(dev); ++ } ++ ++ /* Use call_rcu() here to ensure rcu critical sections have completed ++ * Remembering the driver side flush operation will happen before the ++ * net device is removed. ++ */ ++ old_dev = xchg(&dtab->netdev_map[i], dev); ++ if (old_dev) ++ call_rcu(&old_dev->rcu, __dev_map_entry_free); ++ ++ return 0; ++} ++ ++static int dev_map_update_elem(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ return __dev_map_update_elem(current->nsproxy->net_ns, ++ map, key, value, map_flags); ++} ++ ++static int __dev_map_hash_update_elem(struct net *net, struct bpf_map *map, ++ void *key, void *value, u64 map_flags) ++{ ++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); ++ struct bpf_dtab_netdev *dev, *old_dev; ++ u32 ifindex = *(u32 *)value; ++ u32 idx = *(u32 *)key; ++ unsigned long flags; ++ int err = -EEXIST; ++ ++ if (unlikely(map_flags > BPF_EXIST || !ifindex)) ++ return -EINVAL; ++ ++ spin_lock_irqsave(&dtab->index_lock, flags); ++ ++ old_dev = __dev_map_hash_lookup_elem(map, idx); ++ if (old_dev && (map_flags & BPF_NOEXIST)) ++ goto out_err; ++ ++ dev = __dev_map_alloc_node(net, dtab, ifindex, idx); ++ if (IS_ERR(dev)) { ++ err = PTR_ERR(dev); ++ goto out_err; ++ } ++ ++ if (old_dev) { ++ hlist_del_rcu(&old_dev->index_hlist); ++ } else { ++ if (dtab->items >= dtab->map.max_entries) { ++ spin_unlock_irqrestore(&dtab->index_lock, flags); ++ call_rcu(&dev->rcu, __dev_map_entry_free); ++ return -E2BIG; ++ } ++ dtab->items++; ++ } ++ ++ hlist_add_head_rcu(&dev->index_hlist, ++ dev_map_index_hash(dtab, idx)); ++ spin_unlock_irqrestore(&dtab->index_lock, flags); ++ ++ if (old_dev) ++ call_rcu(&old_dev->rcu, __dev_map_entry_free); ++ ++ return 0; ++ ++out_err: ++ spin_unlock_irqrestore(&dtab->index_lock, flags); ++ return err; ++} ++ ++static int dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ return __dev_map_hash_update_elem(current->nsproxy->net_ns, ++ map, key, value, map_flags); ++} ++ ++const struct bpf_map_ops dev_map_ops = { ++ .map_alloc = dev_map_alloc, ++ .map_free = dev_map_free, ++ .map_get_next_key = dev_map_get_next_key, ++ .map_lookup_elem = dev_map_lookup_elem, ++ .map_update_elem = dev_map_update_elem, ++ .map_delete_elem = dev_map_delete_elem, ++ .map_check_btf = map_check_no_btf, ++}; ++ ++const struct bpf_map_ops dev_map_hash_ops = { ++ .map_alloc = dev_map_alloc, ++ .map_free = dev_map_free, ++ .map_get_next_key = dev_map_hash_get_next_key, ++ .map_lookup_elem = dev_map_hash_lookup_elem, ++ .map_update_elem = dev_map_hash_update_elem, ++ .map_delete_elem = dev_map_hash_delete_elem, ++ .map_check_btf = map_check_no_btf, ++}; ++ ++static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab, ++ struct net_device *netdev) ++{ ++ unsigned long flags; ++ u32 i; ++ ++ spin_lock_irqsave(&dtab->index_lock, flags); ++ for (i = 0; i < dtab->n_buckets; i++) { ++ struct bpf_dtab_netdev *dev; ++ struct hlist_head *head; ++ struct hlist_node *next; ++ ++ head = dev_map_index_hash(dtab, i); ++ ++ hlist_for_each_entry_safe(dev, next, head, index_hlist) { ++ if (netdev != dev->dev) ++ continue; ++ ++ dtab->items--; ++ hlist_del_rcu(&dev->index_hlist); ++ call_rcu(&dev->rcu, __dev_map_entry_free); ++ } ++ } ++ spin_unlock_irqrestore(&dtab->index_lock, flags); ++} ++ ++static int dev_map_notification(struct notifier_block *notifier, ++ ulong event, void *ptr) ++{ ++ struct net_device *netdev = netdev_notifier_info_to_dev(ptr); ++ struct bpf_dtab *dtab; ++ int i; ++ ++ switch (event) { ++ case NETDEV_UNREGISTER: ++ /* This rcu_read_lock/unlock pair is needed because ++ * dev_map_list is an RCU list AND to ensure a delete ++ * operation does not free a netdev_map entry while we ++ * are comparing it against the netdev being unregistered. ++ */ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(dtab, &dev_map_list, list) { ++ if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { ++ dev_map_hash_remove_netdev(dtab, netdev); ++ continue; ++ } ++ ++ for (i = 0; i < dtab->map.max_entries; i++) { ++ struct bpf_dtab_netdev *dev, *odev; ++ ++ dev = READ_ONCE(dtab->netdev_map[i]); ++ if (!dev || netdev != dev->dev) ++ continue; ++ odev = cmpxchg(&dtab->netdev_map[i], dev, NULL); ++ if (dev == odev) ++ call_rcu(&dev->rcu, ++ __dev_map_entry_free); ++ } ++ } ++ rcu_read_unlock(); ++ break; ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block dev_map_notifier = { ++ .notifier_call = dev_map_notification, ++}; ++ ++static int __init dev_map_init(void) ++{ ++ /* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */ ++ BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) != ++ offsetof(struct _bpf_dtab_netdev, dev)); ++ register_netdevice_notifier(&dev_map_notifier); ++ return 0; ++} ++ ++subsys_initcall(dev_map_init); +--- /dev/null ++++ b/kernel/bpf/disasm.c +@@ -0,0 +1,258 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com ++ * Copyright (c) 2016 Facebook ++ */ ++ ++#include ++ ++#include "disasm.h" ++ ++#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x) ++static const char * const func_id_str[] = { ++ __BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN) ++}; ++#undef __BPF_FUNC_STR_FN ++ ++static const char *__func_get_name(const struct bpf_insn_cbs *cbs, ++ const struct bpf_insn *insn, ++ char *buff, size_t len) ++{ ++ BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID); ++ ++ if (insn->src_reg != BPF_PSEUDO_CALL && ++ insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID && ++ func_id_str[insn->imm]) ++ return func_id_str[insn->imm]; ++ ++ if (cbs && cbs->cb_call) ++ return cbs->cb_call(cbs->private_data, insn); ++ ++ if (insn->src_reg == BPF_PSEUDO_CALL) ++ snprintf(buff, len, "%+d", insn->imm); ++ ++ return buff; ++} ++ ++static const char *__func_imm_name(const struct bpf_insn_cbs *cbs, ++ const struct bpf_insn *insn, ++ u64 full_imm, char *buff, size_t len) ++{ ++ if (cbs && cbs->cb_imm) ++ return cbs->cb_imm(cbs->private_data, insn, full_imm); ++ ++ snprintf(buff, len, "0x%llx", (unsigned long long)full_imm); ++ return buff; ++} ++ ++const char *func_id_name(int id) ++{ ++ if (id >= 0 && id < __BPF_FUNC_MAX_ID && func_id_str[id]) ++ return func_id_str[id]; ++ else ++ return "unknown"; ++} ++ ++const char *const bpf_class_string[8] = { ++ [BPF_LD] = "ld", ++ [BPF_LDX] = "ldx", ++ [BPF_ST] = "st", ++ [BPF_STX] = "stx", ++ [BPF_ALU] = "alu", ++ [BPF_JMP] = "jmp", ++ [BPF_JMP32] = "jmp32", ++ [BPF_ALU64] = "alu64", ++}; ++ ++const char *const bpf_alu_string[16] = { ++ [BPF_ADD >> 4] = "+=", ++ [BPF_SUB >> 4] = "-=", ++ [BPF_MUL >> 4] = "*=", ++ [BPF_DIV >> 4] = "/=", ++ [BPF_OR >> 4] = "|=", ++ [BPF_AND >> 4] = "&=", ++ [BPF_LSH >> 4] = "<<=", ++ [BPF_RSH >> 4] = ">>=", ++ [BPF_NEG >> 4] = "neg", ++ [BPF_MOD >> 4] = "%=", ++ [BPF_XOR >> 4] = "^=", ++ [BPF_MOV >> 4] = "=", ++ [BPF_ARSH >> 4] = "s>>=", ++ [BPF_END >> 4] = "endian", ++}; ++ ++static const char *const bpf_ldst_string[] = { ++ [BPF_W >> 3] = "u32", ++ [BPF_H >> 3] = "u16", ++ [BPF_B >> 3] = "u8", ++ [BPF_DW >> 3] = "u64", ++}; ++ ++static const char *const bpf_jmp_string[16] = { ++ [BPF_JA >> 4] = "jmp", ++ [BPF_JEQ >> 4] = "==", ++ [BPF_JGT >> 4] = ">", ++ [BPF_JLT >> 4] = "<", ++ [BPF_JGE >> 4] = ">=", ++ [BPF_JLE >> 4] = "<=", ++ [BPF_JSET >> 4] = "&", ++ [BPF_JNE >> 4] = "!=", ++ [BPF_JSGT >> 4] = "s>", ++ [BPF_JSLT >> 4] = "s<", ++ [BPF_JSGE >> 4] = "s>=", ++ [BPF_JSLE >> 4] = "s<=", ++ [BPF_CALL >> 4] = "call", ++ [BPF_EXIT >> 4] = "exit", ++}; ++ ++static void print_bpf_end_insn(bpf_insn_print_t verbose, ++ void *private_data, ++ const struct bpf_insn *insn) ++{ ++ verbose(private_data, "(%02x) r%d = %s%d r%d\n", ++ insn->code, insn->dst_reg, ++ BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le", ++ insn->imm, insn->dst_reg); ++} ++ ++void print_bpf_insn(const struct bpf_insn_cbs *cbs, ++ const struct bpf_insn *insn, ++ bool allow_ptr_leaks) ++{ ++ const bpf_insn_print_t verbose = cbs->cb_print; ++ u8 class = BPF_CLASS(insn->code); ++ ++ if (class == BPF_ALU || class == BPF_ALU64) { ++ if (BPF_OP(insn->code) == BPF_END) { ++ if (class == BPF_ALU64) ++ verbose(cbs->private_data, "BUG_alu64_%02x\n", insn->code); ++ else ++ print_bpf_end_insn(verbose, cbs->private_data, insn); ++ } else if (BPF_OP(insn->code) == BPF_NEG) { ++ verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n", ++ insn->code, class == BPF_ALU ? 'w' : 'r', ++ insn->dst_reg, class == BPF_ALU ? 'w' : 'r', ++ insn->dst_reg); ++ } else if (BPF_SRC(insn->code) == BPF_X) { ++ verbose(cbs->private_data, "(%02x) %c%d %s %c%d\n", ++ insn->code, class == BPF_ALU ? 'w' : 'r', ++ insn->dst_reg, ++ bpf_alu_string[BPF_OP(insn->code) >> 4], ++ class == BPF_ALU ? 'w' : 'r', ++ insn->src_reg); ++ } else { ++ verbose(cbs->private_data, "(%02x) %c%d %s %d\n", ++ insn->code, class == BPF_ALU ? 'w' : 'r', ++ insn->dst_reg, ++ bpf_alu_string[BPF_OP(insn->code) >> 4], ++ insn->imm); ++ } ++ } else if (class == BPF_STX) { ++ if (BPF_MODE(insn->code) == BPF_MEM) ++ verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n", ++ insn->code, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->dst_reg, ++ insn->off, insn->src_reg); ++ else if (BPF_MODE(insn->code) == BPF_XADD) ++ verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) += r%d\n", ++ insn->code, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->dst_reg, insn->off, ++ insn->src_reg); ++ else ++ verbose(cbs->private_data, "BUG_%02x\n", insn->code); ++ } else if (class == BPF_ST) { ++ if (BPF_MODE(insn->code) != BPF_MEM) { ++ verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); ++ return; ++ } ++ verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", ++ insn->code, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->dst_reg, ++ insn->off, insn->imm); ++ } else if (class == BPF_LDX) { ++ if (BPF_MODE(insn->code) != BPF_MEM) { ++ verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); ++ return; ++ } ++ verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n", ++ insn->code, insn->dst_reg, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->src_reg, insn->off); ++ } else if (class == BPF_LD) { ++ if (BPF_MODE(insn->code) == BPF_ABS) { ++ verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n", ++ insn->code, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->imm); ++ } else if (BPF_MODE(insn->code) == BPF_IND) { ++ verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n", ++ insn->code, ++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], ++ insn->src_reg, insn->imm); ++ } else if (BPF_MODE(insn->code) == BPF_IMM && ++ BPF_SIZE(insn->code) == BPF_DW) { ++ /* At this point, we already made sure that the second ++ * part of the ldimm64 insn is accessible. ++ */ ++ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; ++ bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || ++ insn->src_reg == BPF_PSEUDO_MAP_VALUE; ++ char tmp[64]; ++ ++ if (is_ptr && !allow_ptr_leaks) ++ imm = 0; ++ ++ verbose(cbs->private_data, "(%02x) r%d = %s\n", ++ insn->code, insn->dst_reg, ++ __func_imm_name(cbs, insn, imm, ++ tmp, sizeof(tmp))); ++ } else { ++ verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code); ++ return; ++ } ++ } else if (class == BPF_JMP32 || class == BPF_JMP) { ++ u8 opcode = BPF_OP(insn->code); ++ ++ if (opcode == BPF_CALL) { ++ char tmp[64]; ++ ++ if (insn->src_reg == BPF_PSEUDO_CALL) { ++ verbose(cbs->private_data, "(%02x) call pc%s\n", ++ insn->code, ++ __func_get_name(cbs, insn, ++ tmp, sizeof(tmp))); ++ } else { ++ strcpy(tmp, "unknown"); ++ verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code, ++ __func_get_name(cbs, insn, ++ tmp, sizeof(tmp)), ++ insn->imm); ++ } ++ } else if (insn->code == (BPF_JMP | BPF_JA)) { ++ verbose(cbs->private_data, "(%02x) goto pc%+d\n", ++ insn->code, insn->off); ++ } else if (insn->code == (BPF_JMP | BPF_EXIT)) { ++ verbose(cbs->private_data, "(%02x) exit\n", insn->code); ++ } else if (BPF_SRC(insn->code) == BPF_X) { ++ verbose(cbs->private_data, ++ "(%02x) if %c%d %s %c%d goto pc%+d\n", ++ insn->code, class == BPF_JMP32 ? 'w' : 'r', ++ insn->dst_reg, ++ bpf_jmp_string[BPF_OP(insn->code) >> 4], ++ class == BPF_JMP32 ? 'w' : 'r', ++ insn->src_reg, insn->off); ++ } else { ++ verbose(cbs->private_data, ++ "(%02x) if %c%d %s 0x%x goto pc%+d\n", ++ insn->code, class == BPF_JMP32 ? 'w' : 'r', ++ insn->dst_reg, ++ bpf_jmp_string[BPF_OP(insn->code) >> 4], ++ insn->imm, insn->off); ++ } ++ } else { ++ verbose(cbs->private_data, "(%02x) %s\n", ++ insn->code, bpf_class_string[class]); ++ } ++} +--- /dev/null ++++ b/kernel/bpf/disasm.h +@@ -0,0 +1,40 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com ++ * Copyright (c) 2016 Facebook ++ */ ++ ++#ifndef __BPF_DISASM_H__ ++#define __BPF_DISASM_H__ ++ ++#include ++#include ++#include ++#ifndef __KERNEL__ ++#include ++#include ++#endif ++ ++extern const char *const bpf_alu_string[16]; ++extern const char *const bpf_class_string[8]; ++ ++const char *func_id_name(int id); ++ ++typedef __printf(2, 3) void (*bpf_insn_print_t)(void *private_data, ++ const char *, ...); ++typedef const char *(*bpf_insn_revmap_call_t)(void *private_data, ++ const struct bpf_insn *insn); ++typedef const char *(*bpf_insn_print_imm_t)(void *private_data, ++ const struct bpf_insn *insn, ++ __u64 full_imm); ++ ++struct bpf_insn_cbs { ++ bpf_insn_print_t cb_print; ++ bpf_insn_revmap_call_t cb_call; ++ bpf_insn_print_imm_t cb_imm; ++ void *private_data; ++}; ++ ++void print_bpf_insn(const struct bpf_insn_cbs *cbs, ++ const struct bpf_insn *insn, ++ bool allow_ptr_leaks); ++#endif +--- a/kernel/bpf/hashtab.c ++++ b/kernel/bpf/hashtab.c +@@ -1,147 +1,467 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. +- * +- * This program is distributed in the hope that it will be useful, but +- * WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. ++ * Copyright (c) 2016 Facebook + */ + #include ++#include + #include + #include +-#include ++#include ++#include ++#include ++#include "percpu_freelist.h" ++#include "bpf_lru_list.h" ++#include "map_in_map.h" ++ ++#define HTAB_CREATE_FLAG_MASK \ ++ (BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \ ++ BPF_F_ACCESS_MASK | BPF_F_ZERO_SEED) ++ ++struct bucket { ++ struct hlist_nulls_head head; ++ raw_spinlock_t lock; ++}; + + struct bpf_htab { + struct bpf_map map; +- struct hlist_head *buckets; +- raw_spinlock_t lock; +- u32 count; /* number of elements in this hashtable */ ++ struct bucket *buckets; ++ void *elems; ++ union { ++ struct pcpu_freelist freelist; ++ struct bpf_lru lru; ++ }; ++ struct htab_elem *__percpu *extra_elems; ++ atomic_t count; /* number of elements in this hashtable */ + u32 n_buckets; /* number of hash buckets */ + u32 elem_size; /* size of each element in bytes */ ++ u32 hashrnd; + }; + + /* each htab element is struct htab_elem + key + value */ + struct htab_elem { +- struct hlist_node hash_node; +- struct rcu_head rcu; ++ union { ++ struct hlist_nulls_node hash_node; ++ struct { ++ void *padding; ++ union { ++ struct bpf_htab *htab; ++ struct pcpu_freelist_node fnode; ++ }; ++ }; ++ }; ++ union { ++ struct rcu_head rcu; ++ struct bpf_lru_node lru_node; ++ }; + u32 hash; + char key[0] __aligned(8); + }; + ++static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node); ++ ++static bool htab_is_lru(const struct bpf_htab *htab) ++{ ++ return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || ++ htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; ++} ++ ++static bool htab_is_percpu(const struct bpf_htab *htab) ++{ ++ return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; ++} ++ ++static bool htab_is_prealloc(const struct bpf_htab *htab) ++{ ++ return !(htab->map.map_flags & BPF_F_NO_PREALLOC); ++} ++ ++static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size, ++ void __percpu *pptr) ++{ ++ *(void __percpu **)(l->key + key_size) = pptr; ++} ++ ++static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size) ++{ ++ return *(void __percpu **)(l->key + key_size); ++} ++ ++static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) ++{ ++ return *(void **)(l->key + roundup(map->key_size, 8)); ++} ++ ++static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) ++{ ++ return (struct htab_elem *) (htab->elems + i * htab->elem_size); ++} ++ ++static void htab_free_elems(struct bpf_htab *htab) ++{ ++ int i; ++ ++ if (!htab_is_percpu(htab)) ++ goto free_elems; ++ ++ for (i = 0; i < htab->map.max_entries; i++) { ++ void __percpu *pptr; ++ ++ pptr = htab_elem_get_ptr(get_htab_elem(htab, i), ++ htab->map.key_size); ++ free_percpu(pptr); ++ cond_resched(); ++ } ++free_elems: ++ bpf_map_area_free(htab->elems); ++} ++ ++static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, ++ u32 hash) ++{ ++ struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); ++ struct htab_elem *l; ++ ++ if (node) { ++ l = container_of(node, struct htab_elem, lru_node); ++ memcpy(l->key, key, htab->map.key_size); ++ return l; ++ } ++ ++ return NULL; ++} ++ ++static int prealloc_init(struct bpf_htab *htab) ++{ ++ u32 num_entries = htab->map.max_entries; ++ int err = -ENOMEM, i; ++ ++ if (!htab_is_percpu(htab) && !htab_is_lru(htab)) ++ num_entries += num_possible_cpus(); ++ ++ htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries, ++ htab->map.numa_node); ++ if (!htab->elems) ++ return -ENOMEM; ++ ++ if (!htab_is_percpu(htab)) ++ goto skip_percpu_elems; ++ ++ for (i = 0; i < num_entries; i++) { ++ u32 size = round_up(htab->map.value_size, 8); ++ void __percpu *pptr; ++ ++ pptr = __alloc_percpu_gfp(size, 8, GFP_USER | __GFP_NOWARN); ++ if (!pptr) ++ goto free_elems; ++ htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, ++ pptr); ++ cond_resched(); ++ } ++ ++skip_percpu_elems: ++ if (htab_is_lru(htab)) ++ err = bpf_lru_init(&htab->lru, ++ htab->map.map_flags & BPF_F_NO_COMMON_LRU, ++ offsetof(struct htab_elem, hash) - ++ offsetof(struct htab_elem, lru_node), ++ htab_lru_map_delete_node, ++ htab); ++ else ++ err = pcpu_freelist_init(&htab->freelist); ++ ++ if (err) ++ goto free_elems; ++ ++ if (htab_is_lru(htab)) ++ bpf_lru_populate(&htab->lru, htab->elems, ++ offsetof(struct htab_elem, lru_node), ++ htab->elem_size, num_entries); ++ else ++ pcpu_freelist_populate(&htab->freelist, ++ htab->elems + offsetof(struct htab_elem, fnode), ++ htab->elem_size, num_entries); ++ ++ return 0; ++ ++free_elems: ++ htab_free_elems(htab); ++ return err; ++} ++ ++static void prealloc_destroy(struct bpf_htab *htab) ++{ ++ htab_free_elems(htab); ++ ++ if (htab_is_lru(htab)) ++ bpf_lru_destroy(&htab->lru); ++ else ++ pcpu_freelist_destroy(&htab->freelist); ++} ++ ++static int alloc_extra_elems(struct bpf_htab *htab) ++{ ++ struct htab_elem *__percpu *pptr, *l_new; ++ struct pcpu_freelist_node *l; ++ int cpu; ++ ++ pptr = __alloc_percpu_gfp(sizeof(struct htab_elem *), 8, ++ GFP_USER | __GFP_NOWARN); ++ if (!pptr) ++ return -ENOMEM; ++ ++ for_each_possible_cpu(cpu) { ++ l = pcpu_freelist_pop(&htab->freelist); ++ /* pop will succeed, since prealloc_init() ++ * preallocated extra num_possible_cpus elements ++ */ ++ l_new = container_of(l, struct htab_elem, fnode); ++ *per_cpu_ptr(pptr, cpu) = l_new; ++ } ++ htab->extra_elems = pptr; ++ return 0; ++} ++ + /* Called from syscall */ +-static struct bpf_map *htab_map_alloc(union bpf_attr *attr) ++static int htab_map_alloc_check(union bpf_attr *attr) + { +- struct bpf_htab *htab; +- int err, i; ++ bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); ++ bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || ++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); ++ /* percpu_lru means each cpu has its own LRU list. ++ * it is different from BPF_MAP_TYPE_PERCPU_HASH where ++ * the map's value itself is percpu. percpu_lru has ++ * nothing to do with the map's value. ++ */ ++ bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); ++ bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); ++ bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED); ++ int numa_node = bpf_map_attr_numa_node(attr); ++ ++ BUILD_BUG_ON(offsetof(struct htab_elem, htab) != ++ offsetof(struct htab_elem, hash_node.pprev)); ++ BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) != ++ offsetof(struct htab_elem, hash_node.pprev)); ++ ++ if (lru && !capable(CAP_SYS_ADMIN)) ++ /* LRU implementation is much complicated than other ++ * maps. Hence, limit to CAP_SYS_ADMIN for now. ++ */ ++ return -EPERM; + +- htab = kzalloc(sizeof(*htab), GFP_USER); +- if (!htab) +- return ERR_PTR(-ENOMEM); ++ if (zero_seed && !capable(CAP_SYS_ADMIN)) ++ /* Guard against local DoS, and discourage production use. */ ++ return -EPERM; + +- /* mandatory map attributes */ +- htab->map.key_size = attr->key_size; +- htab->map.value_size = attr->value_size; +- htab->map.max_entries = attr->max_entries; ++ if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK || ++ !bpf_map_flags_access_ok(attr->map_flags)) ++ return -EINVAL; ++ ++ if (!lru && percpu_lru) ++ return -EINVAL; ++ ++ if (lru && !prealloc) ++ return -ENOTSUPP; ++ ++ if (numa_node != NUMA_NO_NODE && (percpu || percpu_lru)) ++ return -EINVAL; + + /* check sanity of attributes. + * value_size == 0 may be allowed in the future to use map as a set + */ +- err = -EINVAL; +- if (htab->map.max_entries == 0 || htab->map.key_size == 0 || +- htab->map.value_size == 0) +- goto free_htab; +- +- /* hash table size must be power of 2 */ +- htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); ++ if (attr->max_entries == 0 || attr->key_size == 0 || ++ attr->value_size == 0) ++ return -EINVAL; + +- err = -E2BIG; +- if (htab->map.key_size > MAX_BPF_STACK) ++ if (attr->key_size > MAX_BPF_STACK) + /* eBPF programs initialize keys on stack, so they cannot be + * larger than max stack size + */ +- goto free_htab; ++ return -E2BIG; + +- if (htab->map.value_size >= (1 << (KMALLOC_SHIFT_MAX - 1)) - ++ if (attr->value_size >= KMALLOC_MAX_SIZE - + MAX_BPF_STACK - sizeof(struct htab_elem)) + /* if value_size is bigger, the user space won't be able to + * access the elements via bpf syscall. This check also makes + * sure that the elem_size doesn't overflow and it's + * kmalloc-able later in htab_map_update_elem() + */ +- goto free_htab; ++ return -E2BIG; ++ ++ return 0; ++} ++ ++static struct bpf_map *htab_map_alloc(union bpf_attr *attr) ++{ ++ bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); ++ bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || ++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); ++ /* percpu_lru means each cpu has its own LRU list. ++ * it is different from BPF_MAP_TYPE_PERCPU_HASH where ++ * the map's value itself is percpu. percpu_lru has ++ * nothing to do with the map's value. ++ */ ++ bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); ++ bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); ++ struct bpf_htab *htab; ++ int err, i; ++ u64 cost; ++ ++ htab = kzalloc(sizeof(*htab), GFP_USER); ++ if (!htab) ++ return ERR_PTR(-ENOMEM); ++ ++ bpf_map_init_from_attr(&htab->map, attr); ++ ++ if (percpu_lru) { ++ /* ensure each CPU's lru list has >=1 elements. ++ * since we are at it, make each lru list has the same ++ * number of elements. ++ */ ++ htab->map.max_entries = roundup(attr->max_entries, ++ num_possible_cpus()); ++ if (htab->map.max_entries < attr->max_entries) ++ htab->map.max_entries = rounddown(attr->max_entries, ++ num_possible_cpus()); ++ } ++ ++ /* hash table size must be power of 2 */ ++ htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); + + htab->elem_size = sizeof(struct htab_elem) + +- round_up(htab->map.key_size, 8) + +- htab->map.value_size; ++ round_up(htab->map.key_size, 8); ++ if (percpu) ++ htab->elem_size += sizeof(void *); ++ else ++ htab->elem_size += round_up(htab->map.value_size, 8); + ++ err = -E2BIG; + /* prevent zero size kmalloc and check for u32 overflow */ + if (htab->n_buckets == 0 || +- htab->n_buckets > U32_MAX / sizeof(struct hlist_head)) ++ htab->n_buckets > U32_MAX / sizeof(struct bucket)) + goto free_htab; + +- if ((u64) htab->n_buckets * sizeof(struct hlist_head) + +- (u64) htab->elem_size * htab->map.max_entries >= +- U32_MAX - PAGE_SIZE) +- /* make sure page count doesn't overflow */ +- goto free_htab; ++ cost = (u64) htab->n_buckets * sizeof(struct bucket) + ++ (u64) htab->elem_size * htab->map.max_entries; + +- htab->map.pages = round_up(htab->n_buckets * sizeof(struct hlist_head) + +- htab->elem_size * htab->map.max_entries, +- PAGE_SIZE) >> PAGE_SHIFT; ++ if (percpu) ++ cost += (u64) round_up(htab->map.value_size, 8) * ++ num_possible_cpus() * htab->map.max_entries; ++ else ++ cost += (u64) htab->elem_size * num_possible_cpus(); ++ ++ /* if map size is larger than memlock limit, reject it */ ++ err = bpf_map_charge_init(&htab->map.memory, cost); ++ if (err) ++ goto free_htab; + + err = -ENOMEM; +- htab->buckets = kmalloc_array(htab->n_buckets, sizeof(struct hlist_head), +- GFP_USER | __GFP_NOWARN); ++ htab->buckets = bpf_map_area_alloc(htab->n_buckets * ++ sizeof(struct bucket), ++ htab->map.numa_node); ++ if (!htab->buckets) ++ goto free_charge; ++ ++ if (htab->map.map_flags & BPF_F_ZERO_SEED) ++ htab->hashrnd = 0; ++ else ++ htab->hashrnd = get_random_int(); + +- if (!htab->buckets) { +- htab->buckets = vmalloc(htab->n_buckets * sizeof(struct hlist_head)); +- if (!htab->buckets) +- goto free_htab; ++ for (i = 0; i < htab->n_buckets; i++) { ++ INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); ++ raw_spin_lock_init(&htab->buckets[i].lock); + } + +- for (i = 0; i < htab->n_buckets; i++) +- INIT_HLIST_HEAD(&htab->buckets[i]); +- +- raw_spin_lock_init(&htab->lock); +- htab->count = 0; ++ if (prealloc) { ++ err = prealloc_init(htab); ++ if (err) ++ goto free_buckets; ++ ++ if (!percpu && !lru) { ++ /* lru itself can remove the least used element, so ++ * there is no need for an extra elem during map_update. ++ */ ++ err = alloc_extra_elems(htab); ++ if (err) ++ goto free_prealloc; ++ } ++ } + + return &htab->map; + ++free_prealloc: ++ prealloc_destroy(htab); ++free_buckets: ++ bpf_map_area_free(htab->buckets); ++free_charge: ++ bpf_map_charge_finish(&htab->map.memory); + free_htab: + kfree(htab); + return ERR_PTR(err); + } + +-static inline u32 htab_map_hash(const void *key, u32 key_len) ++static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) + { +- return jhash(key, key_len, 0); ++ return jhash(key, key_len, hashrnd); + } + +-static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash) ++static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) + { + return &htab->buckets[hash & (htab->n_buckets - 1)]; + } + +-static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash, ++static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) ++{ ++ return &__select_bucket(htab, hash)->head; ++} ++ ++/* this lookup function can only be called with bucket lock taken */ ++static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, + void *key, u32 key_size) + { ++ struct hlist_nulls_node *n; + struct htab_elem *l; + +- hlist_for_each_entry_rcu(l, head, hash_node) ++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) + if (l->hash == hash && !memcmp(&l->key, key, key_size)) + return l; + + return NULL; + } + +-/* Called from syscall or from eBPF program */ +-static void *htab_map_lookup_elem(struct bpf_map *map, void *key) ++/* can be called without bucket lock. it will repeat the loop in ++ * the unlikely event when elements moved from one bucket into another ++ * while link list is being walked ++ */ ++static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, ++ u32 hash, void *key, ++ u32 key_size, u32 n_buckets) ++{ ++ struct hlist_nulls_node *n; ++ struct htab_elem *l; ++ ++again: ++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) ++ if (l->hash == hash && !memcmp(&l->key, key, key_size)) ++ return l; ++ ++ if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) ++ goto again; ++ ++ return NULL; ++} ++ ++/* Called from syscall or from eBPF program directly, so ++ * arguments have to match bpf_map_lookup_elem() exactly. ++ * The return value is adjusted by BPF instructions ++ * in htab_map_gen_lookup(). ++ */ ++static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) + { + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); +- struct hlist_head *head; ++ struct hlist_nulls_head *head; + struct htab_elem *l; + u32 hash, key_size; + +@@ -150,11 +470,18 @@ static void *htab_map_lookup_elem(struct + + key_size = map->key_size; + +- hash = htab_map_hash(key, key_size); ++ hash = htab_map_hash(key, key_size, htab->hashrnd); + + head = select_bucket(htab, hash); + +- l = lookup_elem_raw(head, hash, key, key_size); ++ l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); ++ ++ return l; ++} ++ ++static void *htab_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct htab_elem *l = __htab_map_lookup_elem(map, key); + + if (l) + return l->key + round_up(map->key_size, 8); +@@ -162,33 +489,138 @@ static void *htab_map_lookup_elem(struct + return NULL; + } + ++/* inline bpf_map_lookup_elem() call. ++ * Instead of: ++ * bpf_prog ++ * bpf_map_lookup_elem ++ * map->ops->map_lookup_elem ++ * htab_map_lookup_elem ++ * __htab_map_lookup_elem ++ * do: ++ * bpf_prog ++ * __htab_map_lookup_elem ++ */ ++static u32 htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) ++{ ++ struct bpf_insn *insn = insn_buf; ++ const int ret = BPF_REG_0; ++ ++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, ++ (void *(*)(struct bpf_map *map, void *key))NULL)); ++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, ++ offsetof(struct htab_elem, key) + ++ round_up(map->key_size, 8)); ++ return insn - insn_buf; ++} ++ ++static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, ++ void *key, const bool mark) ++{ ++ struct htab_elem *l = __htab_map_lookup_elem(map, key); ++ ++ if (l) { ++ if (mark) ++ bpf_lru_node_set_ref(&l->lru_node); ++ return l->key + round_up(map->key_size, 8); ++ } ++ ++ return NULL; ++} ++ ++static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ return __htab_lru_map_lookup_elem(map, key, true); ++} ++ ++static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) ++{ ++ return __htab_lru_map_lookup_elem(map, key, false); ++} ++ ++static u32 htab_lru_map_gen_lookup(struct bpf_map *map, ++ struct bpf_insn *insn_buf) ++{ ++ struct bpf_insn *insn = insn_buf; ++ const int ret = BPF_REG_0; ++ const int ref_reg = BPF_REG_1; ++ ++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, ++ (void *(*)(struct bpf_map *map, void *key))NULL)); ++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); ++ *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, ++ offsetof(struct htab_elem, lru_node) + ++ offsetof(struct bpf_lru_node, ref)); ++ *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); ++ *insn++ = BPF_ST_MEM(BPF_B, ret, ++ offsetof(struct htab_elem, lru_node) + ++ offsetof(struct bpf_lru_node, ref), ++ 1); ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, ++ offsetof(struct htab_elem, key) + ++ round_up(map->key_size, 8)); ++ return insn - insn_buf; ++} ++ ++/* It is called from the bpf_lru_list when the LRU needs to delete ++ * older elements from the htab. ++ */ ++static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) ++{ ++ struct bpf_htab *htab = (struct bpf_htab *)arg; ++ struct htab_elem *l = NULL, *tgt_l; ++ struct hlist_nulls_head *head; ++ struct hlist_nulls_node *n; ++ unsigned long flags; ++ struct bucket *b; ++ ++ tgt_l = container_of(node, struct htab_elem, lru_node); ++ b = __select_bucket(htab, tgt_l->hash); ++ head = &b->head; ++ ++ raw_spin_lock_irqsave(&b->lock, flags); ++ ++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) ++ if (l == tgt_l) { ++ hlist_nulls_del_rcu(&l->hash_node); ++ break; ++ } ++ ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ ++ return l == tgt_l; ++} ++ + /* Called from syscall */ + static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) + { + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); +- struct hlist_head *head; ++ struct hlist_nulls_head *head; + struct htab_elem *l, *next_l; + u32 hash, key_size; +- int i; ++ int i = 0; + + WARN_ON_ONCE(!rcu_read_lock_held()); + + key_size = map->key_size; + +- hash = htab_map_hash(key, key_size); ++ if (!key) ++ goto find_first_elem; ++ ++ hash = htab_map_hash(key, key_size, htab->hashrnd); + + head = select_bucket(htab, hash); + + /* lookup the key */ +- l = lookup_elem_raw(head, hash, key, key_size); ++ l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); + +- if (!l) { +- i = 0; ++ if (!l) + goto find_first_elem; +- } + + /* key was found, get next key in the same bucket */ +- next_l = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&l->hash_node)), ++ next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)), + struct htab_elem, hash_node); + + if (next_l) { +@@ -207,7 +639,7 @@ find_first_elem: + head = select_bucket(htab, i); + + /* pick first element in the bucket */ +- next_l = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)), ++ next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)), + struct htab_elem, hash_node); + if (next_l) { + /* if it's not empty, just return it */ +@@ -216,90 +648,491 @@ find_first_elem: + } + } + +- /* itereated over all buckets and all elements */ ++ /* iterated over all buckets and all elements */ + return -ENOENT; + } + ++static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) ++{ ++ if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) ++ free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); ++ kfree(l); ++} ++ ++static void htab_elem_free_rcu(struct rcu_head *head) ++{ ++ struct htab_elem *l = container_of(head, struct htab_elem, rcu); ++ struct bpf_htab *htab = l->htab; ++ ++ htab_elem_free(htab, l); ++} ++ ++static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) ++{ ++ struct bpf_map *map = &htab->map; ++ void *ptr; ++ ++ if (map->ops->map_fd_put_ptr) { ++ ptr = fd_htab_map_get_ptr(map, l); ++ map->ops->map_fd_put_ptr(ptr); ++ } ++} ++ ++static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) ++{ ++ htab_put_fd_value(htab, l); ++ ++ if (htab_is_prealloc(htab)) { ++ __pcpu_freelist_push(&htab->freelist, &l->fnode); ++ } else { ++ atomic_dec(&htab->count); ++ l->htab = htab; ++ call_rcu(&l->rcu, htab_elem_free_rcu); ++ } ++} ++ ++static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, ++ void *value, bool onallcpus) ++{ ++ if (!onallcpus) { ++ /* copy true value_size bytes */ ++ memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); ++ } else { ++ u32 size = round_up(htab->map.value_size, 8); ++ int off = 0, cpu; ++ ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), ++ value + off, size); ++ off += size; ++ } ++ } ++} ++ ++static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, ++ void *value, bool onallcpus) ++{ ++ /* When using prealloc and not setting the initial value on all cpus, ++ * zero-fill element values for other cpus (just as what happens when ++ * not using prealloc). Otherwise, bpf program has no way to ensure ++ * known initial values for cpus other than current one ++ * (onallcpus=false always when coming from bpf prog). ++ */ ++ if (htab_is_prealloc(htab) && !onallcpus) { ++ u32 size = round_up(htab->map.value_size, 8); ++ int current_cpu = raw_smp_processor_id(); ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ if (cpu == current_cpu) ++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value, ++ size); ++ else ++ memset(per_cpu_ptr(pptr, cpu), 0, size); ++ } ++ } else { ++ pcpu_copy_value(htab, pptr, value, onallcpus); ++ } ++} ++ ++static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab) ++{ ++ return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && ++ BITS_PER_LONG == 64; ++} ++ ++static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, ++ void *value, u32 key_size, u32 hash, ++ bool percpu, bool onallcpus, ++ struct htab_elem *old_elem) ++{ ++ u32 size = htab->map.value_size; ++ bool prealloc = htab_is_prealloc(htab); ++ struct htab_elem *l_new, **pl_new; ++ void __percpu *pptr; ++ ++ if (prealloc) { ++ if (old_elem) { ++ /* if we're updating the existing element, ++ * use per-cpu extra elems to avoid freelist_pop/push ++ */ ++ pl_new = this_cpu_ptr(htab->extra_elems); ++ l_new = *pl_new; ++ htab_put_fd_value(htab, old_elem); ++ *pl_new = old_elem; ++ } else { ++ struct pcpu_freelist_node *l; ++ ++ l = __pcpu_freelist_pop(&htab->freelist); ++ if (!l) ++ return ERR_PTR(-E2BIG); ++ l_new = container_of(l, struct htab_elem, fnode); ++ } ++ } else { ++ if (atomic_inc_return(&htab->count) > htab->map.max_entries) ++ if (!old_elem) { ++ /* when map is full and update() is replacing ++ * old element, it's ok to allocate, since ++ * old element will be freed immediately. ++ * Otherwise return an error ++ */ ++ l_new = ERR_PTR(-E2BIG); ++ goto dec_count; ++ } ++ l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN, ++ htab->map.numa_node); ++ if (!l_new) { ++ l_new = ERR_PTR(-ENOMEM); ++ goto dec_count; ++ } ++ check_and_init_map_lock(&htab->map, ++ l_new->key + round_up(key_size, 8)); ++ } ++ ++ memcpy(l_new->key, key, key_size); ++ if (percpu) { ++ size = round_up(size, 8); ++ if (prealloc) { ++ pptr = htab_elem_get_ptr(l_new, key_size); ++ } else { ++ /* alloc_percpu zero-fills */ ++ pptr = __alloc_percpu_gfp(size, 8, ++ GFP_ATOMIC | __GFP_NOWARN); ++ if (!pptr) { ++ kfree(l_new); ++ l_new = ERR_PTR(-ENOMEM); ++ goto dec_count; ++ } ++ } ++ ++ pcpu_init_value(htab, pptr, value, onallcpus); ++ ++ if (!prealloc) ++ htab_elem_set_ptr(l_new, key_size, pptr); ++ } else if (fd_htab_map_needs_adjust(htab)) { ++ size = round_up(size, 8); ++ memcpy(l_new->key + round_up(key_size, 8), value, size); ++ } else { ++ copy_map_value(&htab->map, ++ l_new->key + round_up(key_size, 8), ++ value); ++ } ++ ++ l_new->hash = hash; ++ return l_new; ++dec_count: ++ atomic_dec(&htab->count); ++ return l_new; ++} ++ ++static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, ++ u64 map_flags) ++{ ++ if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) ++ /* elem already exists */ ++ return -EEXIST; ++ ++ if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) ++ /* elem doesn't exist, cannot update it */ ++ return -ENOENT; ++ ++ return 0; ++} ++ + /* Called from syscall or from eBPF program */ + static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, + u64 map_flags) + { + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); +- struct htab_elem *l_new, *l_old; +- struct hlist_head *head; ++ struct htab_elem *l_new = NULL, *l_old; ++ struct hlist_nulls_head *head; + unsigned long flags; +- u32 key_size; ++ struct bucket *b; ++ u32 key_size, hash; + int ret; + +- if (map_flags > BPF_EXIST) ++ if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) + /* unknown flags */ + return -EINVAL; + + WARN_ON_ONCE(!rcu_read_lock_held()); + +- /* allocate new element outside of lock */ +- l_new = kmalloc(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN); +- if (!l_new) +- return -ENOMEM; +- + key_size = map->key_size; + +- memcpy(l_new->key, key, key_size); +- memcpy(l_new->key + round_up(key_size, 8), value, map->value_size); ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ ++ b = __select_bucket(htab, hash); ++ head = &b->head; + +- l_new->hash = htab_map_hash(l_new->key, key_size); ++ if (unlikely(map_flags & BPF_F_LOCK)) { ++ if (unlikely(!map_value_has_spin_lock(map))) ++ return -EINVAL; ++ /* find an element without taking the bucket lock */ ++ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, ++ htab->n_buckets); ++ ret = check_flags(htab, l_old, map_flags); ++ if (ret) ++ return ret; ++ if (l_old) { ++ /* grab the element lock and update value in place */ ++ copy_map_value_locked(map, ++ l_old->key + round_up(key_size, 8), ++ value, false); ++ return 0; ++ } ++ /* fall through, grab the bucket lock and lookup again. ++ * 99.9% chance that the element won't be found, ++ * but second lookup under lock has to be done. ++ */ ++ } + + /* bpf_map_update_elem() can be called in_irq() */ +- raw_spin_lock_irqsave(&htab->lock, flags); ++ raw_spin_lock_irqsave(&b->lock, flags); + +- head = select_bucket(htab, l_new->hash); ++ l_old = lookup_elem_raw(head, hash, key, key_size); + +- l_old = lookup_elem_raw(head, l_new->hash, key, key_size); ++ ret = check_flags(htab, l_old, map_flags); ++ if (ret) ++ goto err; + +- if (!l_old && unlikely(htab->count >= map->max_entries)) { +- /* if elem with this 'key' doesn't exist and we've reached +- * max_entries limit, fail insertion of new elem ++ if (unlikely(l_old && (map_flags & BPF_F_LOCK))) { ++ /* first lookup without the bucket lock didn't find the element, ++ * but second lookup with the bucket lock found it. ++ * This case is highly unlikely, but has to be dealt with: ++ * grab the element lock in addition to the bucket lock ++ * and update element in place + */ +- ret = -E2BIG; ++ copy_map_value_locked(map, ++ l_old->key + round_up(key_size, 8), ++ value, false); ++ ret = 0; + goto err; + } + +- if (l_old && map_flags == BPF_NOEXIST) { +- /* elem already exists */ +- ret = -EEXIST; ++ l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, ++ l_old); ++ if (IS_ERR(l_new)) { ++ /* all pre-allocated elements are in use or memory exhausted */ ++ ret = PTR_ERR(l_new); + goto err; + } + +- if (!l_old && map_flags == BPF_EXIST) { +- /* elem doesn't exist, cannot update it */ +- ret = -ENOENT; +- goto err; ++ /* add new element to the head of the list, so that ++ * concurrent search will find it before old elem ++ */ ++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); ++ if (l_old) { ++ hlist_nulls_del_rcu(&l_old->hash_node); ++ if (!htab_is_prealloc(htab)) ++ free_htab_elem(htab, l_old); + } ++ ret = 0; ++err: ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ return ret; ++} + +- /* add new element to the head of the list, so that concurrent +- * search will find it before old elem ++static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ struct htab_elem *l_new, *l_old = NULL; ++ struct hlist_nulls_head *head; ++ unsigned long flags; ++ struct bucket *b; ++ u32 key_size, hash; ++ int ret; ++ ++ if (unlikely(map_flags > BPF_EXIST)) ++ /* unknown flags */ ++ return -EINVAL; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ key_size = map->key_size; ++ ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ ++ b = __select_bucket(htab, hash); ++ head = &b->head; ++ ++ /* For LRU, we need to alloc before taking bucket's ++ * spinlock because getting free nodes from LRU may need ++ * to remove older elements from htab and this removal ++ * operation will need a bucket lock. + */ +- hlist_add_head_rcu(&l_new->hash_node, head); ++ l_new = prealloc_lru_pop(htab, key, hash); ++ if (!l_new) ++ return -ENOMEM; ++ memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size); ++ ++ /* bpf_map_update_elem() can be called in_irq() */ ++ raw_spin_lock_irqsave(&b->lock, flags); ++ ++ l_old = lookup_elem_raw(head, hash, key, key_size); ++ ++ ret = check_flags(htab, l_old, map_flags); ++ if (ret) ++ goto err; ++ ++ /* add new element to the head of the list, so that ++ * concurrent search will find it before old elem ++ */ ++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); ++ if (l_old) { ++ bpf_lru_node_set_ref(&l_new->lru_node); ++ hlist_nulls_del_rcu(&l_old->hash_node); ++ } ++ ret = 0; ++ ++err: ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ ++ if (ret) ++ bpf_lru_push_free(&htab->lru, &l_new->lru_node); ++ else if (l_old) ++ bpf_lru_push_free(&htab->lru, &l_old->lru_node); ++ ++ return ret; ++} ++ ++static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 map_flags, ++ bool onallcpus) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ struct htab_elem *l_new = NULL, *l_old; ++ struct hlist_nulls_head *head; ++ unsigned long flags; ++ struct bucket *b; ++ u32 key_size, hash; ++ int ret; ++ ++ if (unlikely(map_flags > BPF_EXIST)) ++ /* unknown flags */ ++ return -EINVAL; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ key_size = map->key_size; ++ ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ ++ b = __select_bucket(htab, hash); ++ head = &b->head; ++ ++ /* bpf_map_update_elem() can be called in_irq() */ ++ raw_spin_lock_irqsave(&b->lock, flags); ++ ++ l_old = lookup_elem_raw(head, hash, key, key_size); ++ ++ ret = check_flags(htab, l_old, map_flags); ++ if (ret) ++ goto err; ++ + if (l_old) { +- hlist_del_rcu(&l_old->hash_node); +- kfree_rcu(l_old, rcu); ++ /* per-cpu hash map can update value in-place */ ++ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), ++ value, onallcpus); + } else { +- htab->count++; ++ l_new = alloc_htab_elem(htab, key, value, key_size, ++ hash, true, onallcpus, NULL); ++ if (IS_ERR(l_new)) { ++ ret = PTR_ERR(l_new); ++ goto err; ++ } ++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); + } +- raw_spin_unlock_irqrestore(&htab->lock, flags); ++ ret = 0; ++err: ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ return ret; ++} + +- return 0; ++static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 map_flags, ++ bool onallcpus) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ struct htab_elem *l_new = NULL, *l_old; ++ struct hlist_nulls_head *head; ++ unsigned long flags; ++ struct bucket *b; ++ u32 key_size, hash; ++ int ret; ++ ++ if (unlikely(map_flags > BPF_EXIST)) ++ /* unknown flags */ ++ return -EINVAL; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ key_size = map->key_size; ++ ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ ++ b = __select_bucket(htab, hash); ++ head = &b->head; ++ ++ /* For LRU, we need to alloc before taking bucket's ++ * spinlock because LRU's elem alloc may need ++ * to remove older elem from htab and this removal ++ * operation will need a bucket lock. ++ */ ++ if (map_flags != BPF_EXIST) { ++ l_new = prealloc_lru_pop(htab, key, hash); ++ if (!l_new) ++ return -ENOMEM; ++ } ++ ++ /* bpf_map_update_elem() can be called in_irq() */ ++ raw_spin_lock_irqsave(&b->lock, flags); ++ ++ l_old = lookup_elem_raw(head, hash, key, key_size); ++ ++ ret = check_flags(htab, l_old, map_flags); ++ if (ret) ++ goto err; ++ ++ if (l_old) { ++ bpf_lru_node_set_ref(&l_old->lru_node); ++ ++ /* per-cpu hash map can update value in-place */ ++ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), ++ value, onallcpus); ++ } else { ++ pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), ++ value, onallcpus); ++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); ++ l_new = NULL; ++ } ++ ret = 0; + err: +- raw_spin_unlock_irqrestore(&htab->lock, flags); +- kfree(l_new); ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ if (l_new) ++ bpf_lru_push_free(&htab->lru, &l_new->lru_node); + return ret; + } + ++static int htab_percpu_map_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 map_flags) ++{ ++ return __htab_percpu_map_update_elem(map, key, value, map_flags, false); ++} ++ ++static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 map_flags) ++{ ++ return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, ++ false); ++} ++ + /* Called from syscall or from eBPF program */ + static int htab_map_delete_elem(struct bpf_map *map, void *key) + { + struct bpf_htab *htab = container_of(map, struct bpf_htab, map); +- struct hlist_head *head; ++ struct hlist_nulls_head *head; ++ struct bucket *b; + struct htab_elem *l; + unsigned long flags; + u32 hash, key_size; +@@ -309,22 +1142,54 @@ static int htab_map_delete_elem(struct b + + key_size = map->key_size; + +- hash = htab_map_hash(key, key_size); ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ b = __select_bucket(htab, hash); ++ head = &b->head; + +- raw_spin_lock_irqsave(&htab->lock, flags); ++ raw_spin_lock_irqsave(&b->lock, flags); + +- head = select_bucket(htab, hash); ++ l = lookup_elem_raw(head, hash, key, key_size); ++ ++ if (l) { ++ hlist_nulls_del_rcu(&l->hash_node); ++ free_htab_elem(htab, l); ++ ret = 0; ++ } ++ ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ return ret; ++} ++ ++static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ struct hlist_nulls_head *head; ++ struct bucket *b; ++ struct htab_elem *l; ++ unsigned long flags; ++ u32 hash, key_size; ++ int ret = -ENOENT; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ key_size = map->key_size; ++ ++ hash = htab_map_hash(key, key_size, htab->hashrnd); ++ b = __select_bucket(htab, hash); ++ head = &b->head; ++ ++ raw_spin_lock_irqsave(&b->lock, flags); + + l = lookup_elem_raw(head, hash, key, key_size); + + if (l) { +- hlist_del_rcu(&l->hash_node); +- htab->count--; +- kfree_rcu(l, rcu); ++ hlist_nulls_del_rcu(&l->hash_node); + ret = 0; + } + +- raw_spin_unlock_irqrestore(&htab->lock, flags); ++ raw_spin_unlock_irqrestore(&b->lock, flags); ++ if (l) ++ bpf_lru_push_free(&htab->lru, &l->lru_node); + return ret; + } + +@@ -333,14 +1198,13 @@ static void delete_all_elements(struct b + int i; + + for (i = 0; i < htab->n_buckets; i++) { +- struct hlist_head *head = select_bucket(htab, i); +- struct hlist_node *n; ++ struct hlist_nulls_head *head = select_bucket(htab, i); ++ struct hlist_nulls_node *n; + struct htab_elem *l; + +- hlist_for_each_entry_safe(l, n, head, hash_node) { +- hlist_del_rcu(&l->hash_node); +- htab->count--; +- kfree(l); ++ hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { ++ hlist_nulls_del_rcu(&l->hash_node); ++ htab_elem_free(htab, l); + } + } + } +@@ -357,31 +1221,320 @@ static void htab_map_free(struct bpf_map + */ + synchronize_rcu(); + +- /* some of kfree_rcu() callbacks for elements of this map may not have +- * executed. It's ok. Proceed to free residual elements and map itself ++ /* some of free_htab_elem() callbacks for elements of this map may ++ * not have executed. Wait for them. + */ +- delete_all_elements(htab); +- kvfree(htab->buckets); ++ rcu_barrier(); ++ if (!htab_is_prealloc(htab)) ++ delete_all_elements(htab); ++ else ++ prealloc_destroy(htab); ++ ++ free_percpu(htab->extra_elems); ++ bpf_map_area_free(htab->buckets); + kfree(htab); + } + +-static const struct bpf_map_ops htab_ops = { ++static void htab_map_seq_show_elem(struct bpf_map *map, void *key, ++ struct seq_file *m) ++{ ++ void *value; ++ ++ rcu_read_lock(); ++ ++ value = htab_map_lookup_elem(map, key); ++ if (!value) { ++ rcu_read_unlock(); ++ return; ++ } ++ ++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); ++ seq_puts(m, ": "); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); ++ seq_puts(m, "\n"); ++ ++ rcu_read_unlock(); ++} ++ ++const struct bpf_map_ops htab_map_ops = { ++ .map_alloc_check = htab_map_alloc_check, + .map_alloc = htab_map_alloc, + .map_free = htab_map_free, + .map_get_next_key = htab_map_get_next_key, + .map_lookup_elem = htab_map_lookup_elem, + .map_update_elem = htab_map_update_elem, + .map_delete_elem = htab_map_delete_elem, ++ .map_gen_lookup = htab_map_gen_lookup, ++ .map_seq_show_elem = htab_map_seq_show_elem, + }; + +-static struct bpf_map_type_list htab_type __read_mostly = { +- .ops = &htab_ops, +- .type = BPF_MAP_TYPE_HASH, ++const struct bpf_map_ops htab_lru_map_ops = { ++ .map_alloc_check = htab_map_alloc_check, ++ .map_alloc = htab_map_alloc, ++ .map_free = htab_map_free, ++ .map_get_next_key = htab_map_get_next_key, ++ .map_lookup_elem = htab_lru_map_lookup_elem, ++ .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, ++ .map_update_elem = htab_lru_map_update_elem, ++ .map_delete_elem = htab_lru_map_delete_elem, ++ .map_gen_lookup = htab_lru_map_gen_lookup, ++ .map_seq_show_elem = htab_map_seq_show_elem, + }; + +-static int __init register_htab_map(void) ++/* Called from eBPF program */ ++static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) + { +- bpf_register_map_type(&htab_type); +- return 0; ++ struct htab_elem *l = __htab_map_lookup_elem(map, key); ++ ++ if (l) ++ return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); ++ else ++ return NULL; + } +-late_initcall(register_htab_map); ++ ++static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct htab_elem *l = __htab_map_lookup_elem(map, key); ++ ++ if (l) { ++ bpf_lru_node_set_ref(&l->lru_node); ++ return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); ++ } ++ ++ return NULL; ++} ++ ++int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) ++{ ++ struct htab_elem *l; ++ void __percpu *pptr; ++ int ret = -ENOENT; ++ int cpu, off = 0; ++ u32 size; ++ ++ /* per_cpu areas are zero-filled and bpf programs can only ++ * access 'value_size' of them, so copying rounded areas ++ * will not leak any kernel data ++ */ ++ size = round_up(map->value_size, 8); ++ rcu_read_lock(); ++ l = __htab_map_lookup_elem(map, key); ++ if (!l) ++ goto out; ++ /* We do not mark LRU map element here in order to not mess up ++ * eviction heuristics when user space does a map walk. ++ */ ++ pptr = htab_elem_get_ptr(l, map->key_size); ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(value + off, ++ per_cpu_ptr(pptr, cpu), size); ++ off += size; ++ } ++ ret = 0; ++out: ++ rcu_read_unlock(); ++ return ret; ++} ++ ++int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ int ret; ++ ++ rcu_read_lock(); ++ if (htab_is_lru(htab)) ++ ret = __htab_lru_percpu_map_update_elem(map, key, value, ++ map_flags, true); ++ else ++ ret = __htab_percpu_map_update_elem(map, key, value, map_flags, ++ true); ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, ++ struct seq_file *m) ++{ ++ struct htab_elem *l; ++ void __percpu *pptr; ++ int cpu; ++ ++ rcu_read_lock(); ++ ++ l = __htab_map_lookup_elem(map, key); ++ if (!l) { ++ rcu_read_unlock(); ++ return; ++ } ++ ++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); ++ seq_puts(m, ": {\n"); ++ pptr = htab_elem_get_ptr(l, map->key_size); ++ for_each_possible_cpu(cpu) { ++ seq_printf(m, "\tcpu%d: ", cpu); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, ++ per_cpu_ptr(pptr, cpu), m); ++ seq_puts(m, "\n"); ++ } ++ seq_puts(m, "}\n"); ++ ++ rcu_read_unlock(); ++} ++ ++const struct bpf_map_ops htab_percpu_map_ops = { ++ .map_alloc_check = htab_map_alloc_check, ++ .map_alloc = htab_map_alloc, ++ .map_free = htab_map_free, ++ .map_get_next_key = htab_map_get_next_key, ++ .map_lookup_elem = htab_percpu_map_lookup_elem, ++ .map_update_elem = htab_percpu_map_update_elem, ++ .map_delete_elem = htab_map_delete_elem, ++ .map_seq_show_elem = htab_percpu_map_seq_show_elem, ++}; ++ ++const struct bpf_map_ops htab_lru_percpu_map_ops = { ++ .map_alloc_check = htab_map_alloc_check, ++ .map_alloc = htab_map_alloc, ++ .map_free = htab_map_free, ++ .map_get_next_key = htab_map_get_next_key, ++ .map_lookup_elem = htab_lru_percpu_map_lookup_elem, ++ .map_update_elem = htab_lru_percpu_map_update_elem, ++ .map_delete_elem = htab_lru_map_delete_elem, ++ .map_seq_show_elem = htab_percpu_map_seq_show_elem, ++}; ++ ++static int fd_htab_map_alloc_check(union bpf_attr *attr) ++{ ++ if (attr->value_size != sizeof(u32)) ++ return -EINVAL; ++ return htab_map_alloc_check(attr); ++} ++ ++static void fd_htab_map_free(struct bpf_map *map) ++{ ++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); ++ struct hlist_nulls_node *n; ++ struct hlist_nulls_head *head; ++ struct htab_elem *l; ++ int i; ++ ++ for (i = 0; i < htab->n_buckets; i++) { ++ head = select_bucket(htab, i); ++ ++ hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { ++ void *ptr = fd_htab_map_get_ptr(map, l); ++ ++ map->ops->map_fd_put_ptr(ptr); ++ } ++ } ++ ++ htab_map_free(map); ++} ++ ++/* only called from syscall */ ++int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) ++{ ++ void **ptr; ++ int ret = 0; ++ ++ if (!map->ops->map_fd_sys_lookup_elem) ++ return -ENOTSUPP; ++ ++ rcu_read_lock(); ++ ptr = htab_map_lookup_elem(map, key); ++ if (ptr) ++ *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); ++ else ++ ret = -ENOENT; ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++/* only called from syscall */ ++int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, ++ void *key, void *value, u64 map_flags) ++{ ++ void *ptr; ++ int ret; ++ u32 ufd = *(u32 *)value; ++ ++ ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); ++ if (IS_ERR(ptr)) ++ return PTR_ERR(ptr); ++ ++ ret = htab_map_update_elem(map, key, &ptr, map_flags); ++ if (ret) ++ map->ops->map_fd_put_ptr(ptr); ++ ++ return ret; ++} ++ ++static struct bpf_map *htab_of_map_alloc(union bpf_attr *attr) ++{ ++ struct bpf_map *map, *inner_map_meta; ++ ++ inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); ++ if (IS_ERR(inner_map_meta)) ++ return inner_map_meta; ++ ++ map = htab_map_alloc(attr); ++ if (IS_ERR(map)) { ++ bpf_map_meta_free(inner_map_meta); ++ return map; ++ } ++ ++ map->inner_map_meta = inner_map_meta; ++ ++ return map; ++} ++ ++static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_map **inner_map = htab_map_lookup_elem(map, key); ++ ++ if (!inner_map) ++ return NULL; ++ ++ return READ_ONCE(*inner_map); ++} ++ ++static u32 htab_of_map_gen_lookup(struct bpf_map *map, ++ struct bpf_insn *insn_buf) ++{ ++ struct bpf_insn *insn = insn_buf; ++ const int ret = BPF_REG_0; ++ ++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, ++ (void *(*)(struct bpf_map *map, void *key))NULL)); ++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, ++ offsetof(struct htab_elem, key) + ++ round_up(map->key_size, 8)); ++ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); ++ ++ return insn - insn_buf; ++} ++ ++static void htab_of_map_free(struct bpf_map *map) ++{ ++ bpf_map_meta_free(map->inner_map_meta); ++ fd_htab_map_free(map); ++} ++ ++const struct bpf_map_ops htab_of_maps_map_ops = { ++ .map_alloc_check = fd_htab_map_alloc_check, ++ .map_alloc = htab_of_map_alloc, ++ .map_free = htab_of_map_free, ++ .map_get_next_key = htab_map_get_next_key, ++ .map_lookup_elem = htab_of_map_lookup_elem, ++ .map_delete_elem = htab_map_delete_elem, ++ .map_fd_get_ptr = bpf_map_fd_get_ptr, ++ .map_fd_put_ptr = bpf_map_fd_put_ptr, ++ .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, ++ .map_gen_lookup = htab_of_map_gen_lookup, ++ .map_check_btf = map_check_no_btf, ++}; +--- a/kernel/bpf/helpers.c ++++ b/kernel/bpf/helpers.c +@@ -1,21 +1,18 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. +- * +- * This program is distributed in the hope that it will be useful, but +- * WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. + */ + #include + #include + #include + #include ++#include + #include + #include + #include ++#include ++#include ++ ++#include "../../lib/kstrtox.h" + + /* If kernel subsystem is allowing eBPF programs to call this function, + * inside its own verifier_ops->get_func_proto() callback it should return +@@ -26,48 +23,32 @@ + * if program is allowed to access maps, so check rcu_read_lock_held in + * all three functions. + */ +-static u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) + { +- /* verifier checked that R1 contains a valid pointer to bpf_map +- * and R2 points to a program stack and map->key_size bytes were +- * initialized +- */ +- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; +- void *key = (void *) (unsigned long) r2; +- void *value; +- + WARN_ON_ONCE(!rcu_read_lock_held()); +- +- value = map->ops->map_lookup_elem(map, key); +- +- /* lookup() returns either pointer to element value or NULL +- * which is the meaning of PTR_TO_MAP_VALUE_OR_NULL type +- */ +- return (unsigned long) value; ++ return (unsigned long) map->ops->map_lookup_elem(map, key); + } + + const struct bpf_func_proto bpf_map_lookup_elem_proto = { + .func = bpf_map_lookup_elem, + .gpl_only = false, ++ .pkt_access = true, + .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, + .arg1_type = ARG_CONST_MAP_PTR, + .arg2_type = ARG_PTR_TO_MAP_KEY, + }; + +-static u64 bpf_map_update_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, ++ void *, value, u64, flags) + { +- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; +- void *key = (void *) (unsigned long) r2; +- void *value = (void *) (unsigned long) r3; +- + WARN_ON_ONCE(!rcu_read_lock_held()); +- +- return map->ops->map_update_elem(map, key, value, r4); ++ return map->ops->map_update_elem(map, key, value, flags); + } + + const struct bpf_func_proto bpf_map_update_elem_proto = { + .func = bpf_map_update_elem, + .gpl_only = false, ++ .pkt_access = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_CONST_MAP_PTR, + .arg2_type = ARG_PTR_TO_MAP_KEY, +@@ -75,33 +56,71 @@ const struct bpf_func_proto bpf_map_upda + .arg4_type = ARG_ANYTHING, + }; + +-static u64 bpf_map_delete_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) + { +- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; +- void *key = (void *) (unsigned long) r2; +- + WARN_ON_ONCE(!rcu_read_lock_held()); +- + return map->ops->map_delete_elem(map, key); + } + + const struct bpf_func_proto bpf_map_delete_elem_proto = { + .func = bpf_map_delete_elem, + .gpl_only = false, ++ .pkt_access = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_CONST_MAP_PTR, + .arg2_type = ARG_PTR_TO_MAP_KEY, + }; + ++BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags) ++{ ++ return map->ops->map_push_elem(map, value, flags); ++} ++ ++const struct bpf_func_proto bpf_map_push_elem_proto = { ++ .func = bpf_map_push_elem, ++ .gpl_only = false, ++ .pkt_access = true, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_CONST_MAP_PTR, ++ .arg2_type = ARG_PTR_TO_MAP_VALUE, ++ .arg3_type = ARG_ANYTHING, ++}; ++ ++BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value) ++{ ++ return map->ops->map_pop_elem(map, value); ++} ++ ++const struct bpf_func_proto bpf_map_pop_elem_proto = { ++ .func = bpf_map_pop_elem, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_CONST_MAP_PTR, ++ .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, ++}; ++ ++BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value) ++{ ++ return map->ops->map_peek_elem(map, value); ++} ++ ++const struct bpf_func_proto bpf_map_peek_elem_proto = { ++ .func = bpf_map_peek_elem, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_CONST_MAP_PTR, ++ .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, ++}; ++ + const struct bpf_func_proto bpf_get_prandom_u32_proto = { + .func = bpf_user_rnd_u32, + .gpl_only = false, + .ret_type = RET_INTEGER, + }; + +-static u64 bpf_get_smp_processor_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_0(bpf_get_smp_processor_id) + { +- return raw_smp_processor_id(); ++ return smp_processor_id(); + } + + const struct bpf_func_proto bpf_get_smp_processor_id_proto = { +@@ -110,7 +129,18 @@ const struct bpf_func_proto bpf_get_smp_ + .ret_type = RET_INTEGER, + }; + +-static u64 bpf_ktime_get_ns(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_0(bpf_get_numa_node_id) ++{ ++ return numa_node_id(); ++} ++ ++const struct bpf_func_proto bpf_get_numa_node_id_proto = { ++ .func = bpf_get_numa_node_id, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++}; ++ ++BPF_CALL_0(bpf_ktime_get_ns) + { + /* NMI safe access to clock monotonic */ + return ktime_get_mono_fast_ns(); +@@ -122,11 +152,11 @@ const struct bpf_func_proto bpf_ktime_ge + .ret_type = RET_INTEGER, + }; + +-static u64 bpf_get_current_pid_tgid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_0(bpf_get_current_pid_tgid) + { + struct task_struct *task = current; + +- if (!task) ++ if (unlikely(!task)) + return -EINVAL; + + return (u64) task->tgid << 32 | task->pid; +@@ -138,18 +168,18 @@ const struct bpf_func_proto bpf_get_curr + .ret_type = RET_INTEGER, + }; + +-static u64 bpf_get_current_uid_gid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_0(bpf_get_current_uid_gid) + { + struct task_struct *task = current; + kuid_t uid; + kgid_t gid; + +- if (!task) ++ if (unlikely(!task)) + return -EINVAL; + + current_uid_gid(&uid, &gid); + return (u64) from_kgid(&init_user_ns, gid) << 32 | +- from_kuid(&init_user_ns, uid); ++ from_kuid(&init_user_ns, uid); + } + + const struct bpf_func_proto bpf_get_current_uid_gid_proto = { +@@ -158,22 +188,254 @@ const struct bpf_func_proto bpf_get_curr + .ret_type = RET_INTEGER, + }; + +-static u64 bpf_get_current_comm(u64 r1, u64 size, u64 r3, u64 r4, u64 r5) ++BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size) + { + struct task_struct *task = current; +- char *buf = (char *) (long) r1; + +- if (!task) +- return -EINVAL; ++ if (unlikely(!task)) ++ goto err_clear; ++ ++ strncpy(buf, task->comm, size); + +- strlcpy(buf, task->comm, min_t(size_t, size, sizeof(task->comm))); ++ /* Verifier guarantees that size > 0. For task->comm exceeding ++ * size, guarantee that buf is %NUL-terminated. Unconditionally ++ * done here to save the size test. ++ */ ++ buf[size - 1] = 0; + return 0; ++err_clear: ++ memset(buf, 0, size); ++ return -EINVAL; + } + + const struct bpf_func_proto bpf_get_current_comm_proto = { + .func = bpf_get_current_comm, + .gpl_only = false, + .ret_type = RET_INTEGER, +- .arg1_type = ARG_PTR_TO_STACK, +- .arg2_type = ARG_CONST_STACK_SIZE, ++ .arg1_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg2_type = ARG_CONST_SIZE, ++}; ++ ++#if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) ++ ++static inline void __bpf_spin_lock(struct bpf_spin_lock *lock) ++{ ++ arch_spinlock_t *l = (void *)lock; ++ union { ++ __u32 val; ++ arch_spinlock_t lock; ++ } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED }; ++ ++ compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0"); ++ BUILD_BUG_ON(sizeof(*l) != sizeof(__u32)); ++ BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32)); ++ arch_spin_lock(l); ++} ++ ++static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock) ++{ ++ arch_spinlock_t *l = (void *)lock; ++ ++ arch_spin_unlock(l); ++} ++ ++#else ++ ++static inline void __bpf_spin_lock(struct bpf_spin_lock *lock) ++{ ++ atomic_t *l = (void *)lock; ++ ++ BUILD_BUG_ON(sizeof(*l) != sizeof(*lock)); ++ do { ++ smp_cond_load_relaxed(&l->counter, !VAL); ++ } while (atomic_xchg(l, 1)); ++} ++ ++static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock) ++{ ++ atomic_t *l = (void *)lock; ++ ++ atomic_set_release(l, 0); ++} ++ ++#endif ++ ++static DEFINE_PER_CPU(unsigned long, irqsave_flags); ++ ++notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) ++{ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ __bpf_spin_lock(lock); ++ __this_cpu_write(irqsave_flags, flags); ++ return 0; ++} ++ ++const struct bpf_func_proto bpf_spin_lock_proto = { ++ .func = bpf_spin_lock, ++ .gpl_only = false, ++ .ret_type = RET_VOID, ++ .arg1_type = ARG_PTR_TO_SPIN_LOCK, ++}; ++ ++notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) ++{ ++ unsigned long flags; ++ ++ flags = __this_cpu_read(irqsave_flags); ++ __bpf_spin_unlock(lock); ++ local_irq_restore(flags); ++ return 0; ++} ++ ++const struct bpf_func_proto bpf_spin_unlock_proto = { ++ .func = bpf_spin_unlock, ++ .gpl_only = false, ++ .ret_type = RET_VOID, ++ .arg1_type = ARG_PTR_TO_SPIN_LOCK, ++}; ++ ++void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, ++ bool lock_src) ++{ ++ struct bpf_spin_lock *lock; ++ ++ if (lock_src) ++ lock = src + map->spin_lock_off; ++ else ++ lock = dst + map->spin_lock_off; ++ preempt_disable(); ++ ____bpf_spin_lock(lock); ++ copy_map_value(map, dst, src); ++ ____bpf_spin_unlock(lock); ++ preempt_enable(); ++} ++ ++#define BPF_STRTOX_BASE_MASK 0x1F ++ ++static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags, ++ unsigned long long *res, bool *is_negative) ++{ ++ unsigned int base = flags & BPF_STRTOX_BASE_MASK; ++ const char *cur_buf = buf; ++ size_t cur_len = buf_len; ++ unsigned int consumed; ++ size_t val_len; ++ char str[64]; ++ ++ if (!buf || !buf_len || !res || !is_negative) ++ return -EINVAL; ++ ++ if (base != 0 && base != 8 && base != 10 && base != 16) ++ return -EINVAL; ++ ++ if (flags & ~BPF_STRTOX_BASE_MASK) ++ return -EINVAL; ++ ++ while (cur_buf < buf + buf_len && isspace(*cur_buf)) ++ ++cur_buf; ++ ++ *is_negative = (cur_buf < buf + buf_len && *cur_buf == '-'); ++ if (*is_negative) ++ ++cur_buf; ++ ++ consumed = cur_buf - buf; ++ cur_len -= consumed; ++ if (!cur_len) ++ return -EINVAL; ++ ++ cur_len = min(cur_len, sizeof(str) - 1); ++ memcpy(str, cur_buf, cur_len); ++ str[cur_len] = '\0'; ++ cur_buf = str; ++ ++ cur_buf = _parse_integer_fixup_radix(cur_buf, &base); ++ val_len = _parse_integer(cur_buf, base, res); ++ ++ if (val_len & KSTRTOX_OVERFLOW) ++ return -ERANGE; ++ ++ if (val_len == 0) ++ return -EINVAL; ++ ++ cur_buf += val_len; ++ consumed += cur_buf - str; ++ ++ return consumed; ++} ++ ++static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags, ++ long long *res) ++{ ++ unsigned long long _res; ++ bool is_negative; ++ int err; ++ ++ err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative); ++ if (err < 0) ++ return err; ++ if (is_negative) { ++ if ((long long)-_res > 0) ++ return -ERANGE; ++ *res = -_res; ++ } else { ++ if ((long long)_res < 0) ++ return -ERANGE; ++ *res = _res; ++ } ++ return err; ++} ++ ++BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags, ++ long *, res) ++{ ++ long long _res; ++ int err; ++ ++ err = __bpf_strtoll(buf, buf_len, flags, &_res); ++ if (err < 0) ++ return err; ++ if (_res != (long)_res) ++ return -ERANGE; ++ *res = _res; ++ return err; ++} ++ ++const struct bpf_func_proto bpf_strtol_proto = { ++ .func = bpf_strtol, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_MEM, ++ .arg2_type = ARG_CONST_SIZE, ++ .arg3_type = ARG_ANYTHING, ++ .arg4_type = ARG_PTR_TO_LONG, ++}; ++ ++BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags, ++ unsigned long *, res) ++{ ++ unsigned long long _res; ++ bool is_negative; ++ int err; ++ ++ err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative); ++ if (err < 0) ++ return err; ++ if (is_negative) ++ return -EINVAL; ++ if (_res != (unsigned long)_res) ++ return -ERANGE; ++ *res = _res; ++ return err; ++} ++ ++const struct bpf_func_proto bpf_strtoul_proto = { ++ .func = bpf_strtoul, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_MEM, ++ .arg2_type = ARG_CONST_SIZE, ++ .arg3_type = ARG_ANYTHING, ++ .arg4_type = ARG_PTR_TO_LONG, + }; +--- a/kernel/bpf/inode.c ++++ b/kernel/bpf/inode.c +@@ -1,3 +1,4 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* + * Minimal file system backend for holding eBPF maps and programs, + * used by bpf(2) object pinning. +@@ -5,21 +6,19 @@ + * Authors: + * + * Daniel Borkmann +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU General Public License +- * version 2 as published by the Free Software Foundation. + */ + +-#include ++#include + #include + #include + #include + #include + #include + #include ++#include + #include + #include ++#include + + enum bpf_type { + BPF_TYPE_UNSPEC = 0, +@@ -87,6 +86,7 @@ static struct inode *bpf_get_inode(struc + switch (mode & S_IFMT) { + case S_IFDIR: + case S_IFREG: ++ case S_IFLNK: + break; + default: + return ERR_PTR(-EINVAL); +@@ -119,18 +119,20 @@ static int bpf_inode_type(const struct i + return 0; + } + +-static bool bpf_dname_reserved(const struct dentry *dentry) ++static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode, ++ struct inode *dir) + { +- return strchr(dentry->d_name.name, '.'); ++ d_instantiate(dentry, inode); ++ dget(dentry); ++ ++ dir->i_mtime = CURRENT_TIME; ++ dir->i_ctime = dir->i_mtime; + } + + static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) + { + struct inode *inode; + +- if (bpf_dname_reserved(dentry)) +- return -EPERM; +- + inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR); + if (IS_ERR(inode)) + return PTR_ERR(inode); +@@ -141,30 +143,30 @@ static int bpf_mkdir(struct inode *dir, + inc_nlink(inode); + inc_nlink(dir); + +- d_instantiate(dentry, inode); +- dget(dentry); +- ++ bpf_dentry_finalize(dentry, inode, dir); + return 0; + } + +-static int bpf_mkobj_ops(struct inode *dir, struct dentry *dentry, +- umode_t mode, const struct inode_operations *iops) ++static int bpffs_obj_open(struct inode *inode, struct file *file) + { +- struct inode *inode; ++ return -EIO; ++} + +- if (bpf_dname_reserved(dentry)) +- return -EPERM; ++static const struct file_operations bpffs_obj_fops = { ++ .open = bpffs_obj_open, ++}; + +- inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFREG); ++static int bpf_mkobj_ops(struct inode *dir, struct dentry *dentry, ++ umode_t mode, const struct inode_operations *iops) ++{ ++ struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFREG); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_op = iops; + inode->i_private = dentry->d_fsdata; + +- d_instantiate(dentry, inode); +- dget(dentry); +- ++ bpf_dentry_finalize(dentry, inode, dir); + return 0; + } + +@@ -187,11 +189,48 @@ static int bpf_mkobj(struct inode *dir, + } + } + ++static struct dentry * ++bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) ++{ ++ /* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future ++ * extensions. ++ */ ++ if (strchr(dentry->d_name.name, '.')) ++ return ERR_PTR(-EPERM); ++ ++ return simple_lookup(dir, dentry, flags); ++} ++ ++static int bpf_symlink(struct inode *dir, struct dentry *dentry, ++ const char *target) ++{ ++ char *link = kstrdup(target, GFP_USER | __GFP_NOWARN); ++ struct inode *inode; ++ ++ if (!link) ++ return -ENOMEM; ++ ++ inode = bpf_get_inode(dir->i_sb, dir, S_IRWXUGO | S_IFLNK); ++ if (IS_ERR(inode)) { ++ kfree(link); ++ return PTR_ERR(inode); ++ } ++ ++ inode->i_op = &simple_symlink_inode_operations; ++ inode->i_link = link; ++ ++ bpf_dentry_finalize(dentry, inode, dir); ++ return 0; ++} ++ + static const struct inode_operations bpf_dir_iops = { +- .lookup = simple_lookup, ++ .lookup = bpf_lookup, + .mknod = bpf_mkobj, + .mkdir = bpf_mkdir, ++ .symlink = bpf_symlink, + .rmdir = simple_rmdir, ++ .rename = simple_rename, ++ .link = simple_link, + .unlink = simple_unlink, + }; + +@@ -256,7 +295,7 @@ out: + } + + static void *bpf_obj_do_get(const struct filename *pathname, +- enum bpf_type *type) ++ enum bpf_type *type, int flags) + { + struct inode *inode; + struct path path; +@@ -268,7 +307,7 @@ static void *bpf_obj_do_get(const struct + return ERR_PTR(ret); + + inode = d_backing_inode(path.dentry); +- ret = inode_permission(inode, MAY_WRITE); ++ ret = inode_permission(inode, ACC_MODE(flags)); + if (ret) + goto out; + +@@ -287,18 +326,23 @@ out: + return ERR_PTR(ret); + } + +-int bpf_obj_get_user(const char __user *pathname) ++int bpf_obj_get_user(const char __user *pathname, int flags) + { + enum bpf_type type = BPF_TYPE_UNSPEC; + struct filename *pname; + int ret = -ENOENT; ++ int f_flags; + void *raw; + ++ f_flags = bpf_get_file_flag(flags); ++ if (f_flags < 0) ++ return f_flags; ++ + pname = getname(pathname); + if (IS_ERR(pname)) + return PTR_ERR(pname); + +- raw = bpf_obj_do_get(pname, &type); ++ raw = bpf_obj_do_get(pname, &type, f_flags); + if (IS_ERR(raw)) { + ret = PTR_ERR(raw); + goto out; +@@ -307,7 +351,7 @@ int bpf_obj_get_user(const char __user * + if (type == BPF_TYPE_PROG) + ret = bpf_prog_new_fd(raw); + else if (type == BPF_TYPE_MAP) +- ret = bpf_map_new_fd(raw); ++ ret = bpf_map_new_fd(raw, f_flags); + else + goto out; + +@@ -318,29 +362,131 @@ out: + return ret; + } + +-static void bpf_evict_inode(struct inode *inode) ++static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type) + { +- enum bpf_type type; ++ struct bpf_prog *prog; ++ int ret = inode_permission(inode, MAY_READ); ++ if (ret) ++ return ERR_PTR(ret); ++ ++ if (inode->i_op == &bpf_map_iops) ++ return ERR_PTR(-EINVAL); ++ if (inode->i_op != &bpf_prog_iops) ++ return ERR_PTR(-EACCES); + +- truncate_inode_pages_final(&inode->i_data); +- clear_inode(inode); ++ prog = inode->i_private; + ++ if (!bpf_prog_get_ok(prog, &type, false)) ++ return ERR_PTR(-EINVAL); ++ ++ return bpf_prog_inc(prog); ++} ++ ++struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type) ++{ ++ struct bpf_prog *prog; ++ struct path path; ++ int ret = kern_path(name, LOOKUP_FOLLOW, &path); ++ if (ret) ++ return ERR_PTR(ret); ++ prog = __get_prog_inode(d_backing_inode(path.dentry), type); ++ if (!IS_ERR(prog)) ++ touch_atime(&path); ++ path_put(&path); ++ return prog; ++} ++EXPORT_SYMBOL(bpf_prog_get_type_path); ++ ++/* ++ * Display the mount options in /proc/mounts. ++ */ ++static int bpf_show_options(struct seq_file *m, struct dentry *root) ++{ ++ umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; ++ ++ if (mode != S_IRWXUGO) ++ seq_printf(m, ",mode=%o", mode); ++ return 0; ++} ++ ++static void bpf_destroy_inode_deferred(struct rcu_head *head) ++{ ++ struct inode *inode = container_of(head, struct inode, i_rcu); ++ enum bpf_type type; ++ ++ if (S_ISLNK(inode->i_mode)) ++ kfree(inode->i_link); + if (!bpf_inode_type(inode, &type)) + bpf_any_put(inode->i_private, type); ++ free_inode_nonrcu(inode); ++} ++ ++static void bpf_destroy_inode(struct inode *inode) ++{ ++ call_rcu(&inode->i_rcu, bpf_destroy_inode_deferred); + } + + static const struct super_operations bpf_super_ops = { + .statfs = simple_statfs, + .drop_inode = generic_delete_inode, +- .evict_inode = bpf_evict_inode, ++ .show_options = bpf_show_options, ++ .destroy_inode = bpf_destroy_inode, ++}; ++ ++enum { ++ OPT_MODE, ++ OPT_ERR, ++}; ++ ++static const match_table_t bpf_mount_tokens = { ++ { OPT_MODE, "mode=%o" }, ++ { OPT_ERR, NULL }, ++}; ++ ++struct bpf_mount_opts { ++ umode_t mode; + }; + ++static int bpf_parse_options(char *data, struct bpf_mount_opts *opts) ++{ ++ substring_t args[MAX_OPT_ARGS]; ++ int option, token; ++ char *ptr; ++ ++ opts->mode = S_IRWXUGO; ++ ++ while ((ptr = strsep(&data, ",")) != NULL) { ++ if (!*ptr) ++ continue; ++ ++ token = match_token(ptr, bpf_mount_tokens, args); ++ switch (token) { ++ case OPT_MODE: ++ if (match_octal(&args[0], &option)) ++ return -EINVAL; ++ opts->mode = option & S_IALLUGO; ++ break; ++ /* We might like to report bad mount options here, but ++ * traditionally we've ignored all mount options, so we'd ++ * better continue to ignore non-existing options for bpf. ++ */ ++ } ++ } ++ ++ return 0; ++} ++ + static int bpf_fill_super(struct super_block *sb, void *data, int silent) + { + static struct tree_descr bpf_rfiles[] = { { "" } }; ++ struct bpf_mount_opts opts; + struct inode *inode; + int ret; + ++ ret = bpf_parse_options(data, &opts); ++ if (ret) ++ return ret; ++ + ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); + if (ret) + return ret; +@@ -350,7 +496,7 @@ static int bpf_fill_super(struct super_b + inode = sb->s_root->d_inode; + inode->i_op = &bpf_dir_iops; + inode->i_mode &= ~S_IALLUGO; +- inode->i_mode |= S_ISVTX | S_IRWXUGO; ++ inode->i_mode |= S_ISVTX | opts.mode; + + return 0; + } +@@ -368,8 +514,6 @@ static struct file_system_type bpf_fs_ty + .kill_sb = kill_litter_super, + }; + +-MODULE_ALIAS_FS("bpf"); +- + static int __init bpf_init(void) + { + int ret; +--- /dev/null ++++ b/kernel/bpf/local_storage.c +@@ -0,0 +1,600 @@ ++//SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); ++ ++#ifdef CONFIG_CGROUP_BPF ++ ++#define LOCAL_STORAGE_CREATE_FLAG_MASK \ ++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) ++ ++struct bpf_cgroup_storage_map { ++ struct bpf_map map; ++ ++ spinlock_t lock; ++ struct bpf_prog *prog; ++ struct rb_root root; ++ struct list_head list; ++}; ++ ++static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map) ++{ ++ return container_of(map, struct bpf_cgroup_storage_map, map); ++} ++ ++static int bpf_cgroup_storage_key_cmp( ++ const struct bpf_cgroup_storage_key *key1, ++ const struct bpf_cgroup_storage_key *key2) ++{ ++ if (key1->cgroup_inode_id < key2->cgroup_inode_id) ++ return -1; ++ else if (key1->cgroup_inode_id > key2->cgroup_inode_id) ++ return 1; ++ else if (key1->attach_type < key2->attach_type) ++ return -1; ++ else if (key1->attach_type > key2->attach_type) ++ return 1; ++ return 0; ++} ++ ++static struct bpf_cgroup_storage *cgroup_storage_lookup( ++ struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key, ++ bool locked) ++{ ++ struct rb_root *root = &map->root; ++ struct rb_node *node; ++ ++ if (!locked) ++ spin_lock_bh(&map->lock); ++ ++ node = root->rb_node; ++ while (node) { ++ struct bpf_cgroup_storage *storage; ++ ++ storage = container_of(node, struct bpf_cgroup_storage, node); ++ ++ switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) { ++ case -1: ++ node = node->rb_left; ++ break; ++ case 1: ++ node = node->rb_right; ++ break; ++ default: ++ if (!locked) ++ spin_unlock_bh(&map->lock); ++ return storage; ++ } ++ } ++ ++ if (!locked) ++ spin_unlock_bh(&map->lock); ++ ++ return NULL; ++} ++ ++static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map, ++ struct bpf_cgroup_storage *storage) ++{ ++ struct rb_root *root = &map->root; ++ struct rb_node **new = &(root->rb_node), *parent = NULL; ++ ++ while (*new) { ++ struct bpf_cgroup_storage *this; ++ ++ this = container_of(*new, struct bpf_cgroup_storage, node); ++ ++ parent = *new; ++ switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) { ++ case -1: ++ new = &((*new)->rb_left); ++ break; ++ case 1: ++ new = &((*new)->rb_right); ++ break; ++ default: ++ return -EEXIST; ++ } ++ } ++ ++ rb_link_node(&storage->node, parent, new); ++ rb_insert_color(&storage->node, root); ++ ++ return 0; ++} ++ ++static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key) ++{ ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage *storage; ++ ++ storage = cgroup_storage_lookup(map, key, false); ++ if (!storage) ++ return NULL; ++ ++ return &READ_ONCE(storage->buf)->data[0]; ++} ++ ++static int cgroup_storage_update_elem(struct bpf_map *map, void *_key, ++ void *value, u64 flags) ++{ ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage *storage; ++ struct bpf_storage_buffer *new; ++ ++ if (unlikely(flags & ~(BPF_F_LOCK | BPF_EXIST | BPF_NOEXIST))) ++ return -EINVAL; ++ ++ if (unlikely(flags & BPF_NOEXIST)) ++ return -EINVAL; ++ ++ if (unlikely((flags & BPF_F_LOCK) && ++ !map_value_has_spin_lock(map))) ++ return -EINVAL; ++ ++ storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, ++ key, false); ++ if (!storage) ++ return -ENOENT; ++ ++ if (flags & BPF_F_LOCK) { ++ copy_map_value_locked(map, storage->buf->data, value, false); ++ return 0; ++ } ++ ++ new = kmalloc_node(sizeof(struct bpf_storage_buffer) + ++ map->value_size, ++ __GFP_ZERO | GFP_ATOMIC | __GFP_NOWARN, ++ map->numa_node); ++ if (!new) ++ return -ENOMEM; ++ ++ memcpy(&new->data[0], value, map->value_size); ++ check_and_init_map_lock(map, new->data); ++ ++ new = xchg(&storage->buf, new); ++ kfree_rcu(new, rcu); ++ ++ return 0; ++} ++ ++int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *_key, ++ void *value) ++{ ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage *storage; ++ int cpu, off = 0; ++ u32 size; ++ ++ rcu_read_lock(); ++ storage = cgroup_storage_lookup(map, key, false); ++ if (!storage) { ++ rcu_read_unlock(); ++ return -ENOENT; ++ } ++ ++ /* per_cpu areas are zero-filled and bpf programs can only ++ * access 'value_size' of them, so copying rounded areas ++ * will not leak any kernel data ++ */ ++ size = round_up(_map->value_size, 8); ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(value + off, ++ per_cpu_ptr(storage->percpu_buf, cpu), size); ++ off += size; ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++ ++int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *_key, ++ void *value, u64 map_flags) ++{ ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage *storage; ++ int cpu, off = 0; ++ u32 size; ++ ++ if (map_flags != BPF_ANY && map_flags != BPF_EXIST) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ storage = cgroup_storage_lookup(map, key, false); ++ if (!storage) { ++ rcu_read_unlock(); ++ return -ENOENT; ++ } ++ ++ /* the user space will provide round_up(value_size, 8) bytes that ++ * will be copied into per-cpu area. bpf programs can only access ++ * value_size of it. During lookup the same extra bytes will be ++ * returned or zeros which were zero-filled by percpu_alloc, ++ * so no kernel data leaks possible ++ */ ++ size = round_up(_map->value_size, 8); ++ for_each_possible_cpu(cpu) { ++ bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), ++ value + off, size); ++ off += size; ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++ ++static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key, ++ void *_next_key) ++{ ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage_key *next = _next_key; ++ struct bpf_cgroup_storage *storage; ++ ++ spin_lock_bh(&map->lock); ++ ++ if (list_empty(&map->list)) ++ goto enoent; ++ ++ if (key) { ++ storage = cgroup_storage_lookup(map, key, true); ++ if (!storage) ++ goto enoent; ++ ++ storage = list_next_entry(storage, list); ++ if (!storage) ++ goto enoent; ++ } else { ++ storage = list_first_entry(&map->list, ++ struct bpf_cgroup_storage, list); ++ } ++ ++ spin_unlock_bh(&map->lock); ++ next->attach_type = storage->key.attach_type; ++ next->cgroup_inode_id = storage->key.cgroup_inode_id; ++ return 0; ++ ++enoent: ++ spin_unlock_bh(&map->lock); ++ return -ENOENT; ++} ++ ++static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) ++{ ++ int numa_node = bpf_map_attr_numa_node(attr); ++ struct bpf_cgroup_storage_map *map; ++ struct bpf_map_memory mem; ++ int ret; ++ ++ if (attr->key_size != sizeof(struct bpf_cgroup_storage_key)) ++ return ERR_PTR(-EINVAL); ++ ++ if (attr->value_size == 0) ++ return ERR_PTR(-EINVAL); ++ ++ if (attr->value_size > PAGE_SIZE) ++ return ERR_PTR(-E2BIG); ++ ++ if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK || ++ !bpf_map_flags_access_ok(attr->map_flags)) ++ return ERR_PTR(-EINVAL); ++ ++ if (attr->max_entries) ++ /* max_entries is not used and enforced to be 0 */ ++ return ERR_PTR(-EINVAL); ++ ++ ret = bpf_map_charge_init(&mem, sizeof(struct bpf_cgroup_storage_map)); ++ if (ret < 0) ++ return ERR_PTR(ret); ++ ++ map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map), ++ __GFP_ZERO | GFP_USER, numa_node); ++ if (!map) { ++ bpf_map_charge_finish(&mem); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ bpf_map_charge_move(&map->map.memory, &mem); ++ ++ /* copy mandatory map attributes */ ++ bpf_map_init_from_attr(&map->map, attr); ++ ++ spin_lock_init(&map->lock); ++ map->root = RB_ROOT; ++ INIT_LIST_HEAD(&map->list); ++ ++ return &map->map; ++} ++ ++static void cgroup_storage_map_free(struct bpf_map *_map) ++{ ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ ++ WARN_ON(!RB_EMPTY_ROOT(&map->root)); ++ WARN_ON(!list_empty(&map->list)); ++ ++ kfree(map); ++} ++ ++static int cgroup_storage_delete_elem(struct bpf_map *map, void *key) ++{ ++ return -EINVAL; ++} ++ ++static int cgroup_storage_check_btf(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type) ++{ ++ struct btf_member *m; ++ u32 offset, size; ++ ++ /* Key is expected to be of struct bpf_cgroup_storage_key type, ++ * which is: ++ * struct bpf_cgroup_storage_key { ++ * __u64 cgroup_inode_id; ++ * __u32 attach_type; ++ * }; ++ */ ++ ++ /* ++ * Key_type must be a structure with two fields. ++ */ ++ if (BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT || ++ BTF_INFO_VLEN(key_type->info) != 2) ++ return -EINVAL; ++ ++ /* ++ * The first field must be a 64 bit integer at 0 offset. ++ */ ++ m = (struct btf_member *)(key_type + 1); ++ size = FIELD_SIZEOF(struct bpf_cgroup_storage_key, cgroup_inode_id); ++ if (!btf_member_is_reg_int(btf, key_type, m, 0, size)) ++ return -EINVAL; ++ ++ /* ++ * The second field must be a 32 bit integer at 64 bit offset. ++ */ ++ m++; ++ offset = offsetof(struct bpf_cgroup_storage_key, attach_type); ++ size = FIELD_SIZEOF(struct bpf_cgroup_storage_key, attach_type); ++ if (!btf_member_is_reg_int(btf, key_type, m, offset, size)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *_key, ++ struct seq_file *m) ++{ ++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(map); ++ struct bpf_cgroup_storage_key *key = _key; ++ struct bpf_cgroup_storage *storage; ++ int cpu; ++ ++ rcu_read_lock(); ++ storage = cgroup_storage_lookup(map_to_storage(map), key, false); ++ if (!storage) { ++ rcu_read_unlock(); ++ return; ++ } ++ ++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); ++ stype = cgroup_storage_type(map); ++ if (stype == BPF_CGROUP_STORAGE_SHARED) { ++ seq_puts(m, ": "); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, ++ &READ_ONCE(storage->buf)->data[0], m); ++ seq_puts(m, "\n"); ++ } else { ++ seq_puts(m, ": {\n"); ++ for_each_possible_cpu(cpu) { ++ seq_printf(m, "\tcpu%d: ", cpu); ++ btf_type_seq_show(map->btf, map->btf_value_type_id, ++ per_cpu_ptr(storage->percpu_buf, cpu), ++ m); ++ seq_puts(m, "\n"); ++ } ++ seq_puts(m, "}\n"); ++ } ++ rcu_read_unlock(); ++} ++ ++const struct bpf_map_ops cgroup_storage_map_ops = { ++ .map_alloc = cgroup_storage_map_alloc, ++ .map_free = cgroup_storage_map_free, ++ .map_get_next_key = cgroup_storage_get_next_key, ++ .map_lookup_elem = cgroup_storage_lookup_elem, ++ .map_update_elem = cgroup_storage_update_elem, ++ .map_delete_elem = cgroup_storage_delete_elem, ++ .map_check_btf = cgroup_storage_check_btf, ++ .map_seq_show_elem = cgroup_storage_seq_show_elem, ++}; ++ ++int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map) ++{ ++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ int ret = -EBUSY; ++ ++ spin_lock_bh(&map->lock); ++ ++ if (map->prog && map->prog != prog) ++ goto unlock; ++ if (prog->aux->cgroup_storage[stype] && ++ prog->aux->cgroup_storage[stype] != _map) ++ goto unlock; ++ ++ map->prog = prog; ++ prog->aux->cgroup_storage[stype] = _map; ++ ret = 0; ++unlock: ++ spin_unlock_bh(&map->lock); ++ ++ return ret; ++} ++ ++void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map) ++{ ++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); ++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); ++ ++ spin_lock_bh(&map->lock); ++ if (map->prog == prog) { ++ WARN_ON(prog->aux->cgroup_storage[stype] != _map); ++ map->prog = NULL; ++ prog->aux->cgroup_storage[stype] = NULL; ++ } ++ spin_unlock_bh(&map->lock); ++} ++ ++static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages) ++{ ++ size_t size; ++ ++ if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) { ++ size = sizeof(struct bpf_storage_buffer) + map->value_size; ++ *pages = round_up(sizeof(struct bpf_cgroup_storage) + size, ++ PAGE_SIZE) >> PAGE_SHIFT; ++ } else { ++ size = map->value_size; ++ *pages = round_up(round_up(size, 8) * num_possible_cpus(), ++ PAGE_SIZE) >> PAGE_SHIFT; ++ } ++ ++ return size; ++} ++ ++struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, ++ enum bpf_cgroup_storage_type stype) ++{ ++ struct bpf_cgroup_storage *storage; ++ struct bpf_map *map; ++ gfp_t flags; ++ size_t size; ++ u32 pages; ++ ++ map = prog->aux->cgroup_storage[stype]; ++ if (!map) ++ return NULL; ++ ++ size = bpf_cgroup_storage_calculate_size(map, &pages); ++ ++ if (bpf_map_charge_memlock(map, pages)) ++ return ERR_PTR(-EPERM); ++ ++ storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), ++ __GFP_ZERO | GFP_USER, map->numa_node); ++ if (!storage) ++ goto enomem; ++ ++ flags = __GFP_ZERO | GFP_USER; ++ ++ if (stype == BPF_CGROUP_STORAGE_SHARED) { ++ storage->buf = kmalloc_node(size, flags, map->numa_node); ++ if (!storage->buf) ++ goto enomem; ++ check_and_init_map_lock(map, storage->buf->data); ++ } else { ++ storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags); ++ if (!storage->percpu_buf) ++ goto enomem; ++ } ++ ++ storage->map = (struct bpf_cgroup_storage_map *)map; ++ ++ return storage; ++ ++enomem: ++ bpf_map_uncharge_memlock(map, pages); ++ kfree(storage); ++ return ERR_PTR(-ENOMEM); ++} ++ ++static void free_shared_cgroup_storage_rcu(struct rcu_head *rcu) ++{ ++ struct bpf_cgroup_storage *storage = ++ container_of(rcu, struct bpf_cgroup_storage, rcu); ++ ++ kfree(storage->buf); ++ kfree(storage); ++} ++ ++static void free_percpu_cgroup_storage_rcu(struct rcu_head *rcu) ++{ ++ struct bpf_cgroup_storage *storage = ++ container_of(rcu, struct bpf_cgroup_storage, rcu); ++ ++ free_percpu(storage->percpu_buf); ++ kfree(storage); ++} ++ ++void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) ++{ ++ enum bpf_cgroup_storage_type stype; ++ struct bpf_map *map; ++ u32 pages; ++ ++ if (!storage) ++ return; ++ ++ map = &storage->map->map; ++ ++ bpf_cgroup_storage_calculate_size(map, &pages); ++ bpf_map_uncharge_memlock(map, pages); ++ ++ stype = cgroup_storage_type(map); ++ if (stype == BPF_CGROUP_STORAGE_SHARED) ++ call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); ++ else ++ call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); ++} ++ ++void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, ++ struct cgroup *cgroup, ++ enum bpf_attach_type type) ++{ ++ struct bpf_cgroup_storage_map *map; ++ ++ if (!storage) ++ return; ++ ++ storage->key.attach_type = type; ++ storage->key.cgroup_inode_id = cgroup->kn->id.id; ++ ++ map = storage->map; ++ ++ spin_lock_bh(&map->lock); ++ WARN_ON(cgroup_storage_insert(map, storage)); ++ list_add(&storage->list, &map->list); ++ spin_unlock_bh(&map->lock); ++} ++ ++void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) ++{ ++ struct bpf_cgroup_storage_map *map; ++ struct rb_root *root; ++ ++ if (!storage) ++ return; ++ ++ map = storage->map; ++ ++ spin_lock_bh(&map->lock); ++ root = &map->root; ++ rb_erase(&storage->node, root); ++ ++ list_del(&storage->list); ++ spin_unlock_bh(&map->lock); ++} ++ ++#endif +--- /dev/null ++++ b/kernel/bpf/lpm_trie.c +@@ -0,0 +1,746 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Longest prefix match list implementation ++ * ++ * Copyright (c) 2016,2017 Daniel Mack ++ * Copyright (c) 2016 David Herrmann ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Intermediate node */ ++#define LPM_TREE_NODE_FLAG_IM BIT(0) ++ ++struct lpm_trie_node; ++ ++struct lpm_trie_node { ++ struct rcu_head rcu; ++ struct lpm_trie_node __rcu *child[2]; ++ u32 prefixlen; ++ u32 flags; ++ u8 data[0]; ++}; ++ ++struct lpm_trie { ++ struct bpf_map map; ++ struct lpm_trie_node __rcu *root; ++ size_t n_entries; ++ size_t max_prefixlen; ++ size_t data_size; ++ raw_spinlock_t lock; ++}; ++ ++/* This trie implements a longest prefix match algorithm that can be used to ++ * match IP addresses to a stored set of ranges. ++ * ++ * Data stored in @data of struct bpf_lpm_key and struct lpm_trie_node is ++ * interpreted as big endian, so data[0] stores the most significant byte. ++ * ++ * Match ranges are internally stored in instances of struct lpm_trie_node ++ * which each contain their prefix length as well as two pointers that may ++ * lead to more nodes containing more specific matches. Each node also stores ++ * a value that is defined by and returned to userspace via the update_elem ++ * and lookup functions. ++ * ++ * For instance, let's start with a trie that was created with a prefix length ++ * of 32, so it can be used for IPv4 addresses, and one single element that ++ * matches 192.168.0.0/16. The data array would hence contain ++ * [0xc0, 0xa8, 0x00, 0x00] in big-endian notation. This documentation will ++ * stick to IP-address notation for readability though. ++ * ++ * As the trie is empty initially, the new node (1) will be places as root ++ * node, denoted as (R) in the example below. As there are no other node, both ++ * child pointers are %NULL. ++ * ++ * +----------------+ ++ * | (1) (R) | ++ * | 192.168.0.0/16 | ++ * | value: 1 | ++ * | [0] [1] | ++ * +----------------+ ++ * ++ * Next, let's add a new node (2) matching 192.168.0.0/24. As there is already ++ * a node with the same data and a smaller prefix (ie, a less specific one), ++ * node (2) will become a child of (1). In child index depends on the next bit ++ * that is outside of what (1) matches, and that bit is 0, so (2) will be ++ * child[0] of (1): ++ * ++ * +----------------+ ++ * | (1) (R) | ++ * | 192.168.0.0/16 | ++ * | value: 1 | ++ * | [0] [1] | ++ * +----------------+ ++ * | ++ * +----------------+ ++ * | (2) | ++ * | 192.168.0.0/24 | ++ * | value: 2 | ++ * | [0] [1] | ++ * +----------------+ ++ * ++ * The child[1] slot of (1) could be filled with another node which has bit #17 ++ * (the next bit after the ones that (1) matches on) set to 1. For instance, ++ * 192.168.128.0/24: ++ * ++ * +----------------+ ++ * | (1) (R) | ++ * | 192.168.0.0/16 | ++ * | value: 1 | ++ * | [0] [1] | ++ * +----------------+ ++ * | | ++ * +----------------+ +------------------+ ++ * | (2) | | (3) | ++ * | 192.168.0.0/24 | | 192.168.128.0/24 | ++ * | value: 2 | | value: 3 | ++ * | [0] [1] | | [0] [1] | ++ * +----------------+ +------------------+ ++ * ++ * Let's add another node (4) to the game for 192.168.1.0/24. In order to place ++ * it, node (1) is looked at first, and because (4) of the semantics laid out ++ * above (bit #17 is 0), it would normally be attached to (1) as child[0]. ++ * However, that slot is already allocated, so a new node is needed in between. ++ * That node does not have a value attached to it and it will never be ++ * returned to users as result of a lookup. It is only there to differentiate ++ * the traversal further. It will get a prefix as wide as necessary to ++ * distinguish its two children: ++ * ++ * +----------------+ ++ * | (1) (R) | ++ * | 192.168.0.0/16 | ++ * | value: 1 | ++ * | [0] [1] | ++ * +----------------+ ++ * | | ++ * +----------------+ +------------------+ ++ * | (4) (I) | | (3) | ++ * | 192.168.0.0/23 | | 192.168.128.0/24 | ++ * | value: --- | | value: 3 | ++ * | [0] [1] | | [0] [1] | ++ * +----------------+ +------------------+ ++ * | | ++ * +----------------+ +----------------+ ++ * | (2) | | (5) | ++ * | 192.168.0.0/24 | | 192.168.1.0/24 | ++ * | value: 2 | | value: 5 | ++ * | [0] [1] | | [0] [1] | ++ * +----------------+ +----------------+ ++ * ++ * 192.168.1.1/32 would be a child of (5) etc. ++ * ++ * An intermediate node will be turned into a 'real' node on demand. In the ++ * example above, (4) would be re-used if 192.168.0.0/23 is added to the trie. ++ * ++ * A fully populated trie would have a height of 32 nodes, as the trie was ++ * created with a prefix length of 32. ++ * ++ * The lookup starts at the root node. If the current node matches and if there ++ * is a child that can be used to become more specific, the trie is traversed ++ * downwards. The last node in the traversal that is a non-intermediate one is ++ * returned. ++ */ ++ ++static inline int extract_bit(const u8 *data, size_t index) ++{ ++ return !!(data[index / 8] & (1 << (7 - (index % 8)))); ++} ++ ++/** ++ * longest_prefix_match() - determine the longest prefix ++ * @trie: The trie to get internal sizes from ++ * @node: The node to operate on ++ * @key: The key to compare to @node ++ * ++ * Determine the longest prefix of @node that matches the bits in @key. ++ */ ++static size_t longest_prefix_match(const struct lpm_trie *trie, ++ const struct lpm_trie_node *node, ++ const struct bpf_lpm_trie_key *key) ++{ ++ u32 limit = min(node->prefixlen, key->prefixlen); ++ u32 prefixlen = 0, i = 0; ++ ++ BUILD_BUG_ON(offsetof(struct lpm_trie_node, data) % sizeof(u32)); ++ BUILD_BUG_ON(offsetof(struct bpf_lpm_trie_key, data) % sizeof(u32)); ++ ++#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && defined(CONFIG_64BIT) ++ ++ /* data_size >= 16 has very small probability. ++ * We do not use a loop for optimal code generation. ++ */ ++ if (trie->data_size >= 8) { ++ u64 diff = be64_to_cpu(*(__be64 *)node->data ^ ++ *(__be64 *)key->data); ++ ++ prefixlen = 64 - fls64(diff); ++ if (prefixlen >= limit) ++ return limit; ++ if (diff) ++ return prefixlen; ++ i = 8; ++ } ++#endif ++ ++ while (trie->data_size >= i + 4) { ++ u32 diff = be32_to_cpu(*(__be32 *)&node->data[i] ^ ++ *(__be32 *)&key->data[i]); ++ ++ prefixlen += 32 - fls(diff); ++ if (prefixlen >= limit) ++ return limit; ++ if (diff) ++ return prefixlen; ++ i += 4; ++ } ++ ++ if (trie->data_size >= i + 2) { ++ u16 diff = be16_to_cpu(*(__be16 *)&node->data[i] ^ ++ *(__be16 *)&key->data[i]); ++ ++ prefixlen += 16 - fls(diff); ++ if (prefixlen >= limit) ++ return limit; ++ if (diff) ++ return prefixlen; ++ i += 2; ++ } ++ ++ if (trie->data_size >= i + 1) { ++ prefixlen += 8 - fls(node->data[i] ^ key->data[i]); ++ ++ if (prefixlen >= limit) ++ return limit; ++ } ++ ++ return prefixlen; ++} ++ ++/* Called from syscall or from eBPF program */ ++static void *trie_lookup_elem(struct bpf_map *map, void *_key) ++{ ++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); ++ struct lpm_trie_node *node, *found = NULL; ++ struct bpf_lpm_trie_key *key = _key; ++ ++ /* Start walking the trie from the root node ... */ ++ ++ for (node = rcu_dereference(trie->root); node;) { ++ unsigned int next_bit; ++ size_t matchlen; ++ ++ /* Determine the longest prefix of @node that matches @key. ++ * If it's the maximum possible prefix for this trie, we have ++ * an exact match and can return it directly. ++ */ ++ matchlen = longest_prefix_match(trie, node, key); ++ if (matchlen == trie->max_prefixlen) { ++ found = node; ++ break; ++ } ++ ++ /* If the number of bits that match is smaller than the prefix ++ * length of @node, bail out and return the node we have seen ++ * last in the traversal (ie, the parent). ++ */ ++ if (matchlen < node->prefixlen) ++ break; ++ ++ /* Consider this node as return candidate unless it is an ++ * artificially added intermediate one. ++ */ ++ if (!(node->flags & LPM_TREE_NODE_FLAG_IM)) ++ found = node; ++ ++ /* If the node match is fully satisfied, let's see if we can ++ * become more specific. Determine the next bit in the key and ++ * traverse down. ++ */ ++ next_bit = extract_bit(key->data, node->prefixlen); ++ node = rcu_dereference(node->child[next_bit]); ++ } ++ ++ if (!found) ++ return NULL; ++ ++ return found->data + trie->data_size; ++} ++ ++static struct lpm_trie_node *lpm_trie_node_alloc(const struct lpm_trie *trie, ++ const void *value) ++{ ++ struct lpm_trie_node *node; ++ size_t size = sizeof(struct lpm_trie_node) + trie->data_size; ++ ++ if (value) ++ size += trie->map.value_size; ++ ++ node = kmalloc_node(size, GFP_ATOMIC | __GFP_NOWARN, ++ trie->map.numa_node); ++ if (!node) ++ return NULL; ++ ++ node->flags = 0; ++ ++ if (value) ++ memcpy(node->data + trie->data_size, value, ++ trie->map.value_size); ++ ++ return node; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int trie_update_elem(struct bpf_map *map, ++ void *_key, void *value, u64 flags) ++{ ++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); ++ struct lpm_trie_node *node, *im_node = NULL, *new_node = NULL; ++ struct lpm_trie_node __rcu **slot; ++ struct bpf_lpm_trie_key *key = _key; ++ unsigned long irq_flags; ++ unsigned int next_bit; ++ size_t matchlen = 0; ++ int ret = 0; ++ ++ if (unlikely(flags > BPF_EXIST)) ++ return -EINVAL; ++ ++ if (key->prefixlen > trie->max_prefixlen) ++ return -EINVAL; ++ ++ raw_spin_lock_irqsave(&trie->lock, irq_flags); ++ ++ /* Allocate and fill a new node */ ++ ++ if (trie->n_entries == trie->map.max_entries) { ++ ret = -ENOSPC; ++ goto out; ++ } ++ ++ new_node = lpm_trie_node_alloc(trie, value); ++ if (!new_node) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ trie->n_entries++; ++ ++ new_node->prefixlen = key->prefixlen; ++ RCU_INIT_POINTER(new_node->child[0], NULL); ++ RCU_INIT_POINTER(new_node->child[1], NULL); ++ memcpy(new_node->data, key->data, trie->data_size); ++ ++ /* Now find a slot to attach the new node. To do that, walk the tree ++ * from the root and match as many bits as possible for each node until ++ * we either find an empty slot or a slot that needs to be replaced by ++ * an intermediate node. ++ */ ++ slot = &trie->root; ++ ++ while ((node = rcu_dereference_protected(*slot, ++ lockdep_is_held(&trie->lock)))) { ++ matchlen = longest_prefix_match(trie, node, key); ++ ++ if (node->prefixlen != matchlen || ++ node->prefixlen == key->prefixlen || ++ node->prefixlen == trie->max_prefixlen) ++ break; ++ ++ next_bit = extract_bit(key->data, node->prefixlen); ++ slot = &node->child[next_bit]; ++ } ++ ++ /* If the slot is empty (a free child pointer or an empty root), ++ * simply assign the @new_node to that slot and be done. ++ */ ++ if (!node) { ++ rcu_assign_pointer(*slot, new_node); ++ goto out; ++ } ++ ++ /* If the slot we picked already exists, replace it with @new_node ++ * which already has the correct data array set. ++ */ ++ if (node->prefixlen == matchlen) { ++ new_node->child[0] = node->child[0]; ++ new_node->child[1] = node->child[1]; ++ ++ if (!(node->flags & LPM_TREE_NODE_FLAG_IM)) ++ trie->n_entries--; ++ ++ rcu_assign_pointer(*slot, new_node); ++ kfree_rcu(node, rcu); ++ ++ goto out; ++ } ++ ++ /* If the new node matches the prefix completely, it must be inserted ++ * as an ancestor. Simply insert it between @node and *@slot. ++ */ ++ if (matchlen == key->prefixlen) { ++ next_bit = extract_bit(node->data, matchlen); ++ rcu_assign_pointer(new_node->child[next_bit], node); ++ rcu_assign_pointer(*slot, new_node); ++ goto out; ++ } ++ ++ im_node = lpm_trie_node_alloc(trie, NULL); ++ if (!im_node) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ im_node->prefixlen = matchlen; ++ im_node->flags |= LPM_TREE_NODE_FLAG_IM; ++ memcpy(im_node->data, node->data, trie->data_size); ++ ++ /* Now determine which child to install in which slot */ ++ if (extract_bit(key->data, matchlen)) { ++ rcu_assign_pointer(im_node->child[0], node); ++ rcu_assign_pointer(im_node->child[1], new_node); ++ } else { ++ rcu_assign_pointer(im_node->child[0], new_node); ++ rcu_assign_pointer(im_node->child[1], node); ++ } ++ ++ /* Finally, assign the intermediate node to the determined spot */ ++ rcu_assign_pointer(*slot, im_node); ++ ++out: ++ if (ret) { ++ if (new_node) ++ trie->n_entries--; ++ ++ kfree(new_node); ++ kfree(im_node); ++ } ++ ++ raw_spin_unlock_irqrestore(&trie->lock, irq_flags); ++ ++ return ret; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int trie_delete_elem(struct bpf_map *map, void *_key) ++{ ++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); ++ struct bpf_lpm_trie_key *key = _key; ++ struct lpm_trie_node __rcu **trim, **trim2; ++ struct lpm_trie_node *node, *parent; ++ unsigned long irq_flags; ++ unsigned int next_bit; ++ size_t matchlen = 0; ++ int ret = 0; ++ ++ if (key->prefixlen > trie->max_prefixlen) ++ return -EINVAL; ++ ++ raw_spin_lock_irqsave(&trie->lock, irq_flags); ++ ++ /* Walk the tree looking for an exact key/length match and keeping ++ * track of the path we traverse. We will need to know the node ++ * we wish to delete, and the slot that points to the node we want ++ * to delete. We may also need to know the nodes parent and the ++ * slot that contains it. ++ */ ++ trim = &trie->root; ++ trim2 = trim; ++ parent = NULL; ++ while ((node = rcu_dereference_protected( ++ *trim, lockdep_is_held(&trie->lock)))) { ++ matchlen = longest_prefix_match(trie, node, key); ++ ++ if (node->prefixlen != matchlen || ++ node->prefixlen == key->prefixlen) ++ break; ++ ++ parent = node; ++ trim2 = trim; ++ next_bit = extract_bit(key->data, node->prefixlen); ++ trim = &node->child[next_bit]; ++ } ++ ++ if (!node || node->prefixlen != key->prefixlen || ++ node->prefixlen != matchlen || ++ (node->flags & LPM_TREE_NODE_FLAG_IM)) { ++ ret = -ENOENT; ++ goto out; ++ } ++ ++ trie->n_entries--; ++ ++ /* If the node we are removing has two children, simply mark it ++ * as intermediate and we are done. ++ */ ++ if (rcu_access_pointer(node->child[0]) && ++ rcu_access_pointer(node->child[1])) { ++ node->flags |= LPM_TREE_NODE_FLAG_IM; ++ goto out; ++ } ++ ++ /* If the parent of the node we are about to delete is an intermediate ++ * node, and the deleted node doesn't have any children, we can delete ++ * the intermediate parent as well and promote its other child ++ * up the tree. Doing this maintains the invariant that all ++ * intermediate nodes have exactly 2 children and that there are no ++ * unnecessary intermediate nodes in the tree. ++ */ ++ if (parent && (parent->flags & LPM_TREE_NODE_FLAG_IM) && ++ !node->child[0] && !node->child[1]) { ++ if (node == rcu_access_pointer(parent->child[0])) ++ rcu_assign_pointer( ++ *trim2, rcu_access_pointer(parent->child[1])); ++ else ++ rcu_assign_pointer( ++ *trim2, rcu_access_pointer(parent->child[0])); ++ kfree_rcu(parent, rcu); ++ kfree_rcu(node, rcu); ++ goto out; ++ } ++ ++ /* The node we are removing has either zero or one child. If there ++ * is a child, move it into the removed node's slot then delete ++ * the node. Otherwise just clear the slot and delete the node. ++ */ ++ if (node->child[0]) ++ rcu_assign_pointer(*trim, rcu_access_pointer(node->child[0])); ++ else if (node->child[1]) ++ rcu_assign_pointer(*trim, rcu_access_pointer(node->child[1])); ++ else ++ RCU_INIT_POINTER(*trim, NULL); ++ kfree_rcu(node, rcu); ++ ++out: ++ raw_spin_unlock_irqrestore(&trie->lock, irq_flags); ++ ++ return ret; ++} ++ ++#define LPM_DATA_SIZE_MAX 256 ++#define LPM_DATA_SIZE_MIN 1 ++ ++#define LPM_VAL_SIZE_MAX (KMALLOC_MAX_SIZE - LPM_DATA_SIZE_MAX - \ ++ sizeof(struct lpm_trie_node)) ++#define LPM_VAL_SIZE_MIN 1 ++ ++#define LPM_KEY_SIZE(X) (sizeof(struct bpf_lpm_trie_key) + (X)) ++#define LPM_KEY_SIZE_MAX LPM_KEY_SIZE(LPM_DATA_SIZE_MAX) ++#define LPM_KEY_SIZE_MIN LPM_KEY_SIZE(LPM_DATA_SIZE_MIN) ++ ++#define LPM_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_NUMA_NODE | \ ++ BPF_F_ACCESS_MASK) ++ ++static struct bpf_map *trie_alloc(union bpf_attr *attr) ++{ ++ struct lpm_trie *trie; ++ u64 cost = sizeof(*trie), cost_per_node; ++ int ret; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return ERR_PTR(-EPERM); ++ ++ /* check sanity of attributes */ ++ if (attr->max_entries == 0 || ++ !(attr->map_flags & BPF_F_NO_PREALLOC) || ++ attr->map_flags & ~LPM_CREATE_FLAG_MASK || ++ !bpf_map_flags_access_ok(attr->map_flags) || ++ attr->key_size < LPM_KEY_SIZE_MIN || ++ attr->key_size > LPM_KEY_SIZE_MAX || ++ attr->value_size < LPM_VAL_SIZE_MIN || ++ attr->value_size > LPM_VAL_SIZE_MAX) ++ return ERR_PTR(-EINVAL); ++ ++ trie = kzalloc(sizeof(*trie), GFP_USER | __GFP_NOWARN); ++ if (!trie) ++ return ERR_PTR(-ENOMEM); ++ ++ /* copy mandatory map attributes */ ++ bpf_map_init_from_attr(&trie->map, attr); ++ trie->data_size = attr->key_size - ++ offsetof(struct bpf_lpm_trie_key, data); ++ trie->max_prefixlen = trie->data_size * 8; ++ ++ cost_per_node = sizeof(struct lpm_trie_node) + ++ attr->value_size + trie->data_size; ++ cost += (u64) attr->max_entries * cost_per_node; ++ ++ ret = bpf_map_charge_init(&trie->map.memory, cost); ++ if (ret) ++ goto out_err; ++ ++ raw_spin_lock_init(&trie->lock); ++ ++ return &trie->map; ++out_err: ++ kfree(trie); ++ return ERR_PTR(ret); ++} ++ ++static void trie_free(struct bpf_map *map) ++{ ++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); ++ struct lpm_trie_node __rcu **slot; ++ struct lpm_trie_node *node; ++ ++ /* Wait for outstanding programs to complete ++ * update/lookup/delete/get_next_key and free the trie. ++ */ ++ synchronize_rcu(); ++ ++ /* Always start at the root and walk down to a node that has no ++ * children. Then free that node, nullify its reference in the parent ++ * and start over. ++ */ ++ ++ for (;;) { ++ slot = &trie->root; ++ ++ for (;;) { ++ node = rcu_dereference_protected(*slot, 1); ++ if (!node) ++ goto out; ++ ++ if (rcu_access_pointer(node->child[0])) { ++ slot = &node->child[0]; ++ continue; ++ } ++ ++ if (rcu_access_pointer(node->child[1])) { ++ slot = &node->child[1]; ++ continue; ++ } ++ ++ kfree(node); ++ RCU_INIT_POINTER(*slot, NULL); ++ break; ++ } ++ } ++ ++out: ++ kfree(trie); ++} ++ ++static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key) ++{ ++ struct lpm_trie_node *node, *next_node = NULL, *parent, *search_root; ++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); ++ struct bpf_lpm_trie_key *key = _key, *next_key = _next_key; ++ struct lpm_trie_node **node_stack = NULL; ++ int err = 0, stack_ptr = -1; ++ unsigned int next_bit; ++ size_t matchlen; ++ ++ /* The get_next_key follows postorder. For the 4 node example in ++ * the top of this file, the trie_get_next_key() returns the following ++ * one after another: ++ * 192.168.0.0/24 ++ * 192.168.1.0/24 ++ * 192.168.128.0/24 ++ * 192.168.0.0/16 ++ * ++ * The idea is to return more specific keys before less specific ones. ++ */ ++ ++ /* Empty trie */ ++ search_root = rcu_dereference(trie->root); ++ if (!search_root) ++ return -ENOENT; ++ ++ /* For invalid key, find the leftmost node in the trie */ ++ if (!key || key->prefixlen > trie->max_prefixlen) ++ goto find_leftmost; ++ ++ node_stack = kmalloc_array(trie->max_prefixlen, ++ sizeof(struct lpm_trie_node *), ++ GFP_ATOMIC | __GFP_NOWARN); ++ if (!node_stack) ++ return -ENOMEM; ++ ++ /* Try to find the exact node for the given key */ ++ for (node = search_root; node;) { ++ node_stack[++stack_ptr] = node; ++ matchlen = longest_prefix_match(trie, node, key); ++ if (node->prefixlen != matchlen || ++ node->prefixlen == key->prefixlen) ++ break; ++ ++ next_bit = extract_bit(key->data, node->prefixlen); ++ node = rcu_dereference(node->child[next_bit]); ++ } ++ if (!node || node->prefixlen != key->prefixlen || ++ (node->flags & LPM_TREE_NODE_FLAG_IM)) ++ goto find_leftmost; ++ ++ /* The node with the exactly-matching key has been found, ++ * find the first node in postorder after the matched node. ++ */ ++ node = node_stack[stack_ptr]; ++ while (stack_ptr > 0) { ++ parent = node_stack[stack_ptr - 1]; ++ if (rcu_dereference(parent->child[0]) == node) { ++ search_root = rcu_dereference(parent->child[1]); ++ if (search_root) ++ goto find_leftmost; ++ } ++ if (!(parent->flags & LPM_TREE_NODE_FLAG_IM)) { ++ next_node = parent; ++ goto do_copy; ++ } ++ ++ node = parent; ++ stack_ptr--; ++ } ++ ++ /* did not find anything */ ++ err = -ENOENT; ++ goto free_stack; ++ ++find_leftmost: ++ /* Find the leftmost non-intermediate node, all intermediate nodes ++ * have exact two children, so this function will never return NULL. ++ */ ++ for (node = search_root; node;) { ++ if (node->flags & LPM_TREE_NODE_FLAG_IM) { ++ node = rcu_dereference(node->child[0]); ++ } else { ++ next_node = node; ++ node = rcu_dereference(node->child[0]); ++ if (!node) ++ node = rcu_dereference(next_node->child[1]); ++ } ++ } ++do_copy: ++ next_key->prefixlen = next_node->prefixlen; ++ memcpy((void *)next_key + offsetof(struct bpf_lpm_trie_key, data), ++ next_node->data, trie->data_size); ++free_stack: ++ kfree(node_stack); ++ return err; ++} ++ ++static int trie_check_btf(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type) ++{ ++ /* Keys must have struct bpf_lpm_trie_key embedded. */ ++ return BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT ? ++ -EINVAL : 0; ++} ++ ++const struct bpf_map_ops trie_map_ops = { ++ .map_alloc = trie_alloc, ++ .map_free = trie_free, ++ .map_get_next_key = trie_get_next_key, ++ .map_lookup_elem = trie_lookup_elem, ++ .map_update_elem = trie_update_elem, ++ .map_delete_elem = trie_delete_elem, ++ .map_check_btf = trie_check_btf, ++}; +--- a/kernel/bpf/Makefile ++++ b/kernel/bpf/Makefile +@@ -1,4 +1,23 @@ ++# SPDX-License-Identifier: GPL-2.0 + obj-y := core.o ++ifneq ($(CONFIG_BPF_JIT_ALWAYS_ON),y) ++# ___bpf_prog_run() needs GCSE disabled on x86; see 3193c0836f203 for details ++cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse ++endif ++CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) + +-obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o +-obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o ++obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o ++obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o ++obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ++obj-$(CONFIG_BPF_SYSCALL) += disasm.o ++obj-$(CONFIG_BPF_SYSCALL) += btf.o ++ifeq ($(CONFIG_NET),y) ++obj-$(CONFIG_BPF_SYSCALL) += devmap.o ++endif ++ifeq ($(CONFIG_PERF_EVENTS),y) ++obj-$(CONFIG_BPF_SYSCALL) += stackmap.o ++endif ++obj-$(CONFIG_CGROUP_BPF) += cgroup.o ++ifeq ($(CONFIG_SYSFS),y) ++obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o ++endif +--- /dev/null ++++ b/kernel/bpf/map_in_map.c +@@ -0,0 +1,120 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2017 Facebook ++ */ ++#include ++#include ++ ++#include "map_in_map.h" ++ ++struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) ++{ ++ struct bpf_map *inner_map, *inner_map_meta; ++ u32 inner_map_meta_size; ++ struct fd f; ++ ++ f = fdget(inner_map_ufd); ++ inner_map = __bpf_map_get(f); ++ if (IS_ERR(inner_map)) ++ return inner_map; ++ ++ /* prog_array->owner_prog_type and owner_jited ++ * is a runtime binding. Doing static check alone ++ * in the verifier is not enough. ++ */ ++ if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY || ++ inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || ++ inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { ++ fdput(f); ++ return ERR_PTR(-ENOTSUPP); ++ } ++ ++ /* Does not support >1 level map-in-map */ ++ if (inner_map->inner_map_meta) { ++ fdput(f); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ if (map_value_has_spin_lock(inner_map)) { ++ fdput(f); ++ return ERR_PTR(-ENOTSUPP); ++ } ++ ++ inner_map_meta_size = sizeof(*inner_map_meta); ++ /* In some cases verifier needs to access beyond just base map. */ ++ if (inner_map->ops == &array_map_ops) ++ inner_map_meta_size = sizeof(struct bpf_array); ++ ++ inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER); ++ if (!inner_map_meta) { ++ fdput(f); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ inner_map_meta->map_type = inner_map->map_type; ++ inner_map_meta->key_size = inner_map->key_size; ++ inner_map_meta->value_size = inner_map->value_size; ++ inner_map_meta->map_flags = inner_map->map_flags; ++ inner_map_meta->max_entries = inner_map->max_entries; ++ inner_map_meta->spin_lock_off = inner_map->spin_lock_off; ++ ++ /* Misc members not needed in bpf_map_meta_equal() check. */ ++ inner_map_meta->ops = inner_map->ops; ++ if (inner_map->ops == &array_map_ops) { ++ inner_map_meta->unpriv_array = inner_map->unpriv_array; ++ container_of(inner_map_meta, struct bpf_array, map)->index_mask = ++ container_of(inner_map, struct bpf_array, map)->index_mask; ++ } ++ ++ fdput(f); ++ return inner_map_meta; ++} ++ ++void bpf_map_meta_free(struct bpf_map *map_meta) ++{ ++ kfree(map_meta); ++} ++ ++bool bpf_map_meta_equal(const struct bpf_map *meta0, ++ const struct bpf_map *meta1) ++{ ++ /* No need to compare ops because it is covered by map_type */ ++ return meta0->map_type == meta1->map_type && ++ meta0->key_size == meta1->key_size && ++ meta0->value_size == meta1->value_size && ++ meta0->map_flags == meta1->map_flags && ++ meta0->max_entries == meta1->max_entries; ++} ++ ++void *bpf_map_fd_get_ptr(struct bpf_map *map, ++ struct file *map_file /* not used */, ++ int ufd) ++{ ++ struct bpf_map *inner_map; ++ struct fd f; ++ ++ f = fdget(ufd); ++ inner_map = __bpf_map_get(f); ++ if (IS_ERR(inner_map)) ++ return inner_map; ++ ++ if (bpf_map_meta_equal(map->inner_map_meta, inner_map)) ++ inner_map = bpf_map_inc(inner_map, false); ++ else ++ inner_map = ERR_PTR(-EINVAL); ++ ++ fdput(f); ++ return inner_map; ++} ++ ++void bpf_map_fd_put_ptr(void *ptr) ++{ ++ /* ptr->ops->map_free() has to go through one ++ * rcu grace period by itself. ++ */ ++ bpf_map_put(ptr); ++} ++ ++u32 bpf_map_fd_sys_lookup_elem(void *ptr) ++{ ++ return ((struct bpf_map *)ptr)->id; ++} +--- /dev/null ++++ b/kernel/bpf/map_in_map.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright (c) 2017 Facebook ++ */ ++#ifndef __MAP_IN_MAP_H__ ++#define __MAP_IN_MAP_H__ ++ ++#include ++ ++struct file; ++struct bpf_map; ++ ++struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd); ++void bpf_map_meta_free(struct bpf_map *map_meta); ++bool bpf_map_meta_equal(const struct bpf_map *meta0, ++ const struct bpf_map *meta1); ++void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file, ++ int ufd); ++void bpf_map_fd_put_ptr(void *ptr); ++u32 bpf_map_fd_sys_lookup_elem(void *ptr); ++ ++#endif +--- /dev/null ++++ b/kernel/bpf/percpu_freelist.c +@@ -0,0 +1,118 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2016 Facebook ++ */ ++#include "percpu_freelist.h" ++ ++int pcpu_freelist_init(struct pcpu_freelist *s) ++{ ++ int cpu; ++ ++ s->freelist = alloc_percpu(struct pcpu_freelist_head); ++ if (!s->freelist) ++ return -ENOMEM; ++ ++ for_each_possible_cpu(cpu) { ++ struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu); ++ ++ raw_spin_lock_init(&head->lock); ++ head->first = NULL; ++ } ++ return 0; ++} ++ ++void pcpu_freelist_destroy(struct pcpu_freelist *s) ++{ ++ free_percpu(s->freelist); ++} ++ ++static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, ++ struct pcpu_freelist_node *node) ++{ ++ raw_spin_lock(&head->lock); ++ node->next = head->first; ++ head->first = node; ++ raw_spin_unlock(&head->lock); ++} ++ ++void __pcpu_freelist_push(struct pcpu_freelist *s, ++ struct pcpu_freelist_node *node) ++{ ++ struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist); ++ ++ ___pcpu_freelist_push(head, node); ++} ++ ++void pcpu_freelist_push(struct pcpu_freelist *s, ++ struct pcpu_freelist_node *node) ++{ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ __pcpu_freelist_push(s, node); ++ local_irq_restore(flags); ++} ++ ++void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size, ++ u32 nr_elems) ++{ ++ struct pcpu_freelist_head *head; ++ unsigned long flags; ++ int i, cpu, pcpu_entries; ++ ++ pcpu_entries = nr_elems / num_possible_cpus() + 1; ++ i = 0; ++ ++ /* disable irq to workaround lockdep false positive ++ * in bpf usage pcpu_freelist_populate() will never race ++ * with pcpu_freelist_push() ++ */ ++ local_irq_save(flags); ++ for_each_possible_cpu(cpu) { ++again: ++ head = per_cpu_ptr(s->freelist, cpu); ++ ___pcpu_freelist_push(head, buf); ++ i++; ++ buf += elem_size; ++ if (i == nr_elems) ++ break; ++ if (i % pcpu_entries) ++ goto again; ++ } ++ local_irq_restore(flags); ++} ++ ++struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s) ++{ ++ struct pcpu_freelist_head *head; ++ struct pcpu_freelist_node *node; ++ int orig_cpu, cpu; ++ ++ orig_cpu = cpu = raw_smp_processor_id(); ++ while (1) { ++ head = per_cpu_ptr(s->freelist, cpu); ++ raw_spin_lock(&head->lock); ++ node = head->first; ++ if (node) { ++ head->first = node->next; ++ raw_spin_unlock(&head->lock); ++ return node; ++ } ++ raw_spin_unlock(&head->lock); ++ cpu = cpumask_next(cpu, cpu_possible_mask); ++ if (cpu >= nr_cpu_ids) ++ cpu = 0; ++ if (cpu == orig_cpu) ++ return NULL; ++ } ++} ++ ++struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s) ++{ ++ struct pcpu_freelist_node *ret; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ ret = __pcpu_freelist_pop(s); ++ local_irq_restore(flags); ++ return ret; ++} +--- /dev/null ++++ b/kernel/bpf/percpu_freelist.h +@@ -0,0 +1,32 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* Copyright (c) 2016 Facebook ++ */ ++#ifndef __PERCPU_FREELIST_H__ ++#define __PERCPU_FREELIST_H__ ++#include ++#include ++ ++struct pcpu_freelist_head { ++ struct pcpu_freelist_node *first; ++ raw_spinlock_t lock; ++}; ++ ++struct pcpu_freelist { ++ struct pcpu_freelist_head __percpu *freelist; ++}; ++ ++struct pcpu_freelist_node { ++ struct pcpu_freelist_node *next; ++}; ++ ++/* pcpu_freelist_* do spin_lock_irqsave. */ ++void pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *); ++struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *); ++/* __pcpu_freelist_* do spin_lock only. caller must disable irqs. */ ++void __pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *); ++struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *); ++void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size, ++ u32 nr_elems); ++int pcpu_freelist_init(struct pcpu_freelist *); ++void pcpu_freelist_destroy(struct pcpu_freelist *s); ++#endif +--- /dev/null ++++ b/kernel/bpf/queue_stack_maps.c +@@ -0,0 +1,289 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * queue_stack_maps.c: BPF queue and stack maps ++ * ++ * Copyright (c) 2018 Politecnico di Torino ++ */ ++#include ++#include ++#include ++#include ++#include "percpu_freelist.h" ++ ++#define QUEUE_STACK_CREATE_FLAG_MASK \ ++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) ++ ++struct bpf_queue_stack { ++ struct bpf_map map; ++ raw_spinlock_t lock; ++ u32 head, tail; ++ u32 size; /* max_entries + 1 */ ++ ++ char elements[0] __aligned(8); ++}; ++ ++static struct bpf_queue_stack *bpf_queue_stack(struct bpf_map *map) ++{ ++ return container_of(map, struct bpf_queue_stack, map); ++} ++ ++static bool queue_stack_map_is_empty(struct bpf_queue_stack *qs) ++{ ++ return qs->head == qs->tail; ++} ++ ++static bool queue_stack_map_is_full(struct bpf_queue_stack *qs) ++{ ++ u32 head = qs->head + 1; ++ ++ if (unlikely(head >= qs->size)) ++ head = 0; ++ ++ return head == qs->tail; ++} ++ ++/* Called from syscall */ ++static int queue_stack_map_alloc_check(union bpf_attr *attr) ++{ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ /* check sanity of attributes */ ++ if (attr->max_entries == 0 || attr->key_size != 0 || ++ attr->value_size == 0 || ++ attr->map_flags & ~QUEUE_STACK_CREATE_FLAG_MASK || ++ !bpf_map_flags_access_ok(attr->map_flags)) ++ return -EINVAL; ++ ++ if (attr->value_size > KMALLOC_MAX_SIZE) ++ /* if value_size is bigger, the user space won't be able to ++ * access the elements. ++ */ ++ return -E2BIG; ++ ++ return 0; ++} ++ ++static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr) ++{ ++ int ret, numa_node = bpf_map_attr_numa_node(attr); ++ struct bpf_map_memory mem = {0}; ++ struct bpf_queue_stack *qs; ++ u64 size, queue_size, cost; ++ ++ size = (u64) attr->max_entries + 1; ++ cost = queue_size = sizeof(*qs) + size * attr->value_size; ++ ++ ret = bpf_map_charge_init(&mem, cost); ++ if (ret < 0) ++ return ERR_PTR(ret); ++ ++ qs = bpf_map_area_alloc(queue_size, numa_node); ++ if (!qs) { ++ bpf_map_charge_finish(&mem); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ memset(qs, 0, sizeof(*qs)); ++ ++ bpf_map_init_from_attr(&qs->map, attr); ++ ++ bpf_map_charge_move(&qs->map.memory, &mem); ++ qs->size = size; ++ ++ raw_spin_lock_init(&qs->lock); ++ ++ return &qs->map; ++} ++ ++/* Called when map->refcnt goes to zero, either from workqueue or from syscall */ ++static void queue_stack_map_free(struct bpf_map *map) ++{ ++ struct bpf_queue_stack *qs = bpf_queue_stack(map); ++ ++ /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, ++ * so the programs (can be more than one that used this map) were ++ * disconnected from events. Wait for outstanding critical sections in ++ * these programs to complete ++ */ ++ synchronize_rcu(); ++ ++ bpf_map_area_free(qs); ++} ++ ++static int __queue_map_get(struct bpf_map *map, void *value, bool delete) ++{ ++ struct bpf_queue_stack *qs = bpf_queue_stack(map); ++ unsigned long flags; ++ int err = 0; ++ void *ptr; ++ ++ raw_spin_lock_irqsave(&qs->lock, flags); ++ ++ if (queue_stack_map_is_empty(qs)) { ++ memset(value, 0, qs->map.value_size); ++ err = -ENOENT; ++ goto out; ++ } ++ ++ ptr = &qs->elements[qs->tail * qs->map.value_size]; ++ memcpy(value, ptr, qs->map.value_size); ++ ++ if (delete) { ++ if (unlikely(++qs->tail >= qs->size)) ++ qs->tail = 0; ++ } ++ ++out: ++ raw_spin_unlock_irqrestore(&qs->lock, flags); ++ return err; ++} ++ ++ ++static int __stack_map_get(struct bpf_map *map, void *value, bool delete) ++{ ++ struct bpf_queue_stack *qs = bpf_queue_stack(map); ++ unsigned long flags; ++ int err = 0; ++ void *ptr; ++ u32 index; ++ ++ raw_spin_lock_irqsave(&qs->lock, flags); ++ ++ if (queue_stack_map_is_empty(qs)) { ++ memset(value, 0, qs->map.value_size); ++ err = -ENOENT; ++ goto out; ++ } ++ ++ index = qs->head - 1; ++ if (unlikely(index >= qs->size)) ++ index = qs->size - 1; ++ ++ ptr = &qs->elements[index * qs->map.value_size]; ++ memcpy(value, ptr, qs->map.value_size); ++ ++ if (delete) ++ qs->head = index; ++ ++out: ++ raw_spin_unlock_irqrestore(&qs->lock, flags); ++ return err; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int queue_map_peek_elem(struct bpf_map *map, void *value) ++{ ++ return __queue_map_get(map, value, false); ++} ++ ++/* Called from syscall or from eBPF program */ ++static int stack_map_peek_elem(struct bpf_map *map, void *value) ++{ ++ return __stack_map_get(map, value, false); ++} ++ ++/* Called from syscall or from eBPF program */ ++static int queue_map_pop_elem(struct bpf_map *map, void *value) ++{ ++ return __queue_map_get(map, value, true); ++} ++ ++/* Called from syscall or from eBPF program */ ++static int stack_map_pop_elem(struct bpf_map *map, void *value) ++{ ++ return __stack_map_get(map, value, true); ++} ++ ++/* Called from syscall or from eBPF program */ ++static int queue_stack_map_push_elem(struct bpf_map *map, void *value, ++ u64 flags) ++{ ++ struct bpf_queue_stack *qs = bpf_queue_stack(map); ++ unsigned long irq_flags; ++ int err = 0; ++ void *dst; ++ ++ /* BPF_EXIST is used to force making room for a new element in case the ++ * map is full ++ */ ++ bool replace = (flags & BPF_EXIST); ++ ++ /* Check supported flags for queue and stack maps */ ++ if (flags & BPF_NOEXIST || flags > BPF_EXIST) ++ return -EINVAL; ++ ++ raw_spin_lock_irqsave(&qs->lock, irq_flags); ++ ++ if (queue_stack_map_is_full(qs)) { ++ if (!replace) { ++ err = -E2BIG; ++ goto out; ++ } ++ /* advance tail pointer to overwrite oldest element */ ++ if (unlikely(++qs->tail >= qs->size)) ++ qs->tail = 0; ++ } ++ ++ dst = &qs->elements[qs->head * qs->map.value_size]; ++ memcpy(dst, value, qs->map.value_size); ++ ++ if (unlikely(++qs->head >= qs->size)) ++ qs->head = 0; ++ ++out: ++ raw_spin_unlock_irqrestore(&qs->lock, irq_flags); ++ return err; ++} ++ ++/* Called from syscall or from eBPF program */ ++static void *queue_stack_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ return NULL; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int queue_stack_map_update_elem(struct bpf_map *map, void *key, ++ void *value, u64 flags) ++{ ++ return -EINVAL; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int queue_stack_map_delete_elem(struct bpf_map *map, void *key) ++{ ++ return -EINVAL; ++} ++ ++/* Called from syscall */ ++static int queue_stack_map_get_next_key(struct bpf_map *map, void *key, ++ void *next_key) ++{ ++ return -EINVAL; ++} ++ ++const struct bpf_map_ops queue_map_ops = { ++ .map_alloc_check = queue_stack_map_alloc_check, ++ .map_alloc = queue_stack_map_alloc, ++ .map_free = queue_stack_map_free, ++ .map_lookup_elem = queue_stack_map_lookup_elem, ++ .map_update_elem = queue_stack_map_update_elem, ++ .map_delete_elem = queue_stack_map_delete_elem, ++ .map_push_elem = queue_stack_map_push_elem, ++ .map_pop_elem = queue_map_pop_elem, ++ .map_peek_elem = queue_map_peek_elem, ++ .map_get_next_key = queue_stack_map_get_next_key, ++}; ++ ++const struct bpf_map_ops stack_map_ops = { ++ .map_alloc_check = queue_stack_map_alloc_check, ++ .map_alloc = queue_stack_map_alloc, ++ .map_free = queue_stack_map_free, ++ .map_lookup_elem = queue_stack_map_lookup_elem, ++ .map_update_elem = queue_stack_map_update_elem, ++ .map_delete_elem = queue_stack_map_delete_elem, ++ .map_push_elem = queue_stack_map_push_elem, ++ .map_pop_elem = stack_map_pop_elem, ++ .map_peek_elem = stack_map_peek_elem, ++ .map_get_next_key = queue_stack_map_get_next_key, ++}; +--- /dev/null ++++ b/kernel/bpf/stackmap.c +@@ -0,0 +1,634 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* Copyright (c) 2016 Facebook ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "percpu_freelist.h" ++ ++#define STACK_CREATE_FLAG_MASK \ ++ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \ ++ BPF_F_STACK_BUILD_ID) ++ ++struct stack_map_bucket { ++ struct pcpu_freelist_node fnode; ++ u32 hash; ++ u32 nr; ++ u64 data[]; ++}; ++ ++struct bpf_stack_map { ++ struct bpf_map map; ++ void *elems; ++ struct pcpu_freelist freelist; ++ u32 n_buckets; ++ struct stack_map_bucket *buckets[]; ++}; ++ ++/* irq_work to run up_read() for build_id lookup in nmi context */ ++struct stack_map_irq_work { ++ struct irq_work irq_work; ++ struct rw_semaphore *sem; ++}; ++ ++static void do_up_read(struct irq_work *entry) ++{ ++ struct stack_map_irq_work *work; ++ ++ work = container_of(entry, struct stack_map_irq_work, irq_work); ++ up_read_non_owner(work->sem); ++ work->sem = NULL; ++} ++ ++static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work); ++ ++static inline bool stack_map_use_build_id(struct bpf_map *map) ++{ ++ return (map->map_flags & BPF_F_STACK_BUILD_ID); ++} ++ ++static inline int stack_map_data_size(struct bpf_map *map) ++{ ++ return stack_map_use_build_id(map) ? ++ sizeof(struct bpf_stack_build_id) : sizeof(u64); ++} ++ ++static int prealloc_elems_and_freelist(struct bpf_stack_map *smap) ++{ ++ u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size; ++ int err; ++ ++ smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries, ++ smap->map.numa_node); ++ if (!smap->elems) ++ return -ENOMEM; ++ ++ err = pcpu_freelist_init(&smap->freelist); ++ if (err) ++ goto free_elems; ++ ++ pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size, ++ smap->map.max_entries); ++ return 0; ++ ++free_elems: ++ bpf_map_area_free(smap->elems); ++ return err; ++} ++ ++/* Called from syscall */ ++static struct bpf_map *stack_map_alloc(union bpf_attr *attr) ++{ ++ u32 value_size = attr->value_size; ++ struct bpf_stack_map *smap; ++ struct bpf_map_memory mem; ++ u64 cost, n_buckets; ++ int err; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return ERR_PTR(-EPERM); ++ ++ if (attr->map_flags & ~STACK_CREATE_FLAG_MASK) ++ return ERR_PTR(-EINVAL); ++ ++ /* check sanity of attributes */ ++ if (attr->max_entries == 0 || attr->key_size != 4 || ++ value_size < 8 || value_size % 8) ++ return ERR_PTR(-EINVAL); ++ ++ BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64)); ++ if (attr->map_flags & BPF_F_STACK_BUILD_ID) { ++ if (value_size % sizeof(struct bpf_stack_build_id) || ++ value_size / sizeof(struct bpf_stack_build_id) ++ > sysctl_perf_event_max_stack) ++ return ERR_PTR(-EINVAL); ++ } else if (value_size / 8 > sysctl_perf_event_max_stack) ++ return ERR_PTR(-EINVAL); ++ ++ /* hash table size must be power of 2 */ ++ n_buckets = roundup_pow_of_two(attr->max_entries); ++ if (!n_buckets) ++ return ERR_PTR(-E2BIG); ++ ++ cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap); ++ cost += n_buckets * (value_size + sizeof(struct stack_map_bucket)); ++ err = bpf_map_charge_init(&mem, cost); ++ if (err) ++ return ERR_PTR(err); ++ ++ smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr)); ++ if (!smap) { ++ bpf_map_charge_finish(&mem); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ bpf_map_init_from_attr(&smap->map, attr); ++ smap->map.value_size = value_size; ++ smap->n_buckets = n_buckets; ++ ++ err = get_callchain_buffers(sysctl_perf_event_max_stack); ++ if (err) ++ goto free_charge; ++ ++ err = prealloc_elems_and_freelist(smap); ++ if (err) ++ goto put_buffers; ++ ++ bpf_map_charge_move(&smap->map.memory, &mem); ++ ++ return &smap->map; ++ ++put_buffers: ++ put_callchain_buffers(); ++free_charge: ++ bpf_map_charge_finish(&mem); ++ bpf_map_area_free(smap); ++ return ERR_PTR(err); ++} ++ ++#define BPF_BUILD_ID 3 ++/* ++ * Parse build id from the note segment. This logic can be shared between ++ * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are ++ * identical. ++ */ ++static inline int stack_map_parse_build_id(void *page_addr, ++ unsigned char *build_id, ++ void *note_start, ++ Elf32_Word note_size) ++{ ++ Elf32_Word note_offs = 0, new_offs; ++ ++ /* check for overflow */ ++ if (note_start < page_addr || note_start + note_size < note_start) ++ return -EINVAL; ++ ++ /* only supports note that fits in the first page */ ++ if (note_start + note_size > page_addr + PAGE_SIZE) ++ return -EINVAL; ++ ++ while (note_offs + sizeof(Elf32_Nhdr) < note_size) { ++ Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); ++ ++ if (nhdr->n_type == BPF_BUILD_ID && ++ nhdr->n_namesz == sizeof("GNU") && ++ nhdr->n_descsz > 0 && ++ nhdr->n_descsz <= BPF_BUILD_ID_SIZE) { ++ memcpy(build_id, ++ note_start + note_offs + ++ ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), ++ nhdr->n_descsz); ++ memset(build_id + nhdr->n_descsz, 0, ++ BPF_BUILD_ID_SIZE - nhdr->n_descsz); ++ return 0; ++ } ++ new_offs = note_offs + sizeof(Elf32_Nhdr) + ++ ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); ++ if (new_offs <= note_offs) /* overflow */ ++ break; ++ note_offs = new_offs; ++ } ++ return -EINVAL; ++} ++ ++/* Parse build ID from 32-bit ELF */ ++static int stack_map_get_build_id_32(void *page_addr, ++ unsigned char *build_id) ++{ ++ Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; ++ Elf32_Phdr *phdr; ++ int i; ++ ++ /* only supports phdr that fits in one page */ ++ if (ehdr->e_phnum > ++ (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr)) ++ return -EINVAL; ++ ++ phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr)); ++ ++ for (i = 0; i < ehdr->e_phnum; ++i) ++ if (phdr[i].p_type == PT_NOTE) ++ return stack_map_parse_build_id(page_addr, build_id, ++ page_addr + phdr[i].p_offset, ++ phdr[i].p_filesz); ++ return -EINVAL; ++} ++ ++/* Parse build ID from 64-bit ELF */ ++static int stack_map_get_build_id_64(void *page_addr, ++ unsigned char *build_id) ++{ ++ Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; ++ Elf64_Phdr *phdr; ++ int i; ++ ++ /* only supports phdr that fits in one page */ ++ if (ehdr->e_phnum > ++ (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr)) ++ return -EINVAL; ++ ++ phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr)); ++ ++ for (i = 0; i < ehdr->e_phnum; ++i) ++ if (phdr[i].p_type == PT_NOTE) ++ return stack_map_parse_build_id(page_addr, build_id, ++ page_addr + phdr[i].p_offset, ++ phdr[i].p_filesz); ++ return -EINVAL; ++} ++ ++/* Parse build ID of ELF file mapped to vma */ ++static int stack_map_get_build_id(struct vm_area_struct *vma, ++ unsigned char *build_id) ++{ ++ Elf32_Ehdr *ehdr; ++ struct page *page; ++ void *page_addr; ++ int ret; ++ ++ /* only works for page backed storage */ ++ if (!vma->vm_file) ++ return -EINVAL; ++ ++ page = find_get_page(vma->vm_file->f_mapping, 0); ++ if (!page) ++ return -EFAULT; /* page not mapped */ ++ ++ ret = -EINVAL; ++ page_addr = kmap_atomic(page); ++ ehdr = (Elf32_Ehdr *)page_addr; ++ ++ /* compare magic x7f "ELF" */ ++ if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) ++ goto out; ++ ++ /* only support executable file and shared object file */ ++ if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) ++ goto out; ++ ++ if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) ++ ret = stack_map_get_build_id_32(page_addr, build_id); ++ else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) ++ ret = stack_map_get_build_id_64(page_addr, build_id); ++out: ++ kunmap_atomic(page_addr); ++ put_page(page); ++ return ret; ++} ++ ++static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, ++ u64 *ips, u32 trace_nr, bool user) ++{ ++ int i; ++ struct vm_area_struct *vma; ++ bool irq_work_busy = false; ++ struct stack_map_irq_work *work = NULL; ++ ++ if (irqs_disabled()) { ++ work = this_cpu_ptr(&up_read_work); ++ if (work->irq_work.flags & IRQ_WORK_BUSY) ++ /* cannot queue more up_read, fallback */ ++ irq_work_busy = true; ++ } ++ ++ /* ++ * We cannot do up_read() when the irq is disabled, because of ++ * risk to deadlock with rq_lock. To do build_id lookup when the ++ * irqs are disabled, we need to run up_read() in irq_work. We use ++ * a percpu variable to do the irq_work. If the irq_work is ++ * already used by another lookup, we fall back to report ips. ++ * ++ * Same fallback is used for kernel stack (!user) on a stackmap ++ * with build_id. ++ */ ++ if (!user || !current || !current->mm || irq_work_busy || ++ down_read_trylock(¤t->mm->mmap_sem) == 0) { ++ /* cannot access current->mm, fall back to ips */ ++ for (i = 0; i < trace_nr; i++) { ++ id_offs[i].status = BPF_STACK_BUILD_ID_IP; ++ id_offs[i].ip = ips[i]; ++ memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); ++ } ++ return; ++ } ++ ++ for (i = 0; i < trace_nr; i++) { ++ vma = find_vma(current->mm, ips[i]); ++ if (!vma || stack_map_get_build_id(vma, id_offs[i].build_id)) { ++ /* per entry fall back to ips */ ++ id_offs[i].status = BPF_STACK_BUILD_ID_IP; ++ id_offs[i].ip = ips[i]; ++ memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); ++ continue; ++ } ++ id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i] ++ - vma->vm_start; ++ id_offs[i].status = BPF_STACK_BUILD_ID_VALID; ++ } ++ ++ if (!work) { ++ up_read(¤t->mm->mmap_sem); ++ } else { ++ work->sem = ¤t->mm->mmap_sem; ++ irq_work_queue(&work->irq_work); ++ /* ++ * The irq_work will release the mmap_sem with ++ * up_read_non_owner(). The rwsem_release() is called ++ * here to release the lock from lockdep's perspective. ++ */ ++ rwsem_release(¤t->mm->mmap_sem.dep_map, 1, _RET_IP_); ++ } ++} ++ ++BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, ++ u64, flags) ++{ ++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); ++ struct perf_callchain_entry *trace; ++ struct stack_map_bucket *bucket, *new_bucket, *old_bucket; ++ u32 max_depth = map->value_size / stack_map_data_size(map); ++ /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */ ++ u32 init_nr = sysctl_perf_event_max_stack - max_depth; ++ u32 skip = flags & BPF_F_SKIP_FIELD_MASK; ++ u32 hash, id, trace_nr, trace_len; ++ bool user = flags & BPF_F_USER_STACK; ++ bool kernel = !user; ++ u64 *ips; ++ bool hash_matches; ++ ++ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | ++ BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) ++ return -EINVAL; ++ ++ trace = get_perf_callchain(regs, init_nr, kernel, user, ++ sysctl_perf_event_max_stack, false, false); ++ ++ if (unlikely(!trace)) ++ /* couldn't fetch the stack trace */ ++ return -EFAULT; ++ ++ /* get_perf_callchain() guarantees that trace->nr >= init_nr ++ * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth ++ */ ++ trace_nr = trace->nr - init_nr; ++ ++ if (trace_nr <= skip) ++ /* skipping more than usable stack trace */ ++ return -EFAULT; ++ ++ trace_nr -= skip; ++ trace_len = trace_nr * sizeof(u64); ++ ips = trace->ip + skip + init_nr; ++ hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); ++ id = hash & (smap->n_buckets - 1); ++ bucket = READ_ONCE(smap->buckets[id]); ++ ++ hash_matches = bucket && bucket->hash == hash; ++ /* fast cmp */ ++ if (hash_matches && flags & BPF_F_FAST_STACK_CMP) ++ return id; ++ ++ if (stack_map_use_build_id(map)) { ++ /* for build_id+offset, pop a bucket before slow cmp */ ++ new_bucket = (struct stack_map_bucket *) ++ pcpu_freelist_pop(&smap->freelist); ++ if (unlikely(!new_bucket)) ++ return -ENOMEM; ++ new_bucket->nr = trace_nr; ++ stack_map_get_build_id_offset( ++ (struct bpf_stack_build_id *)new_bucket->data, ++ ips, trace_nr, user); ++ trace_len = trace_nr * sizeof(struct bpf_stack_build_id); ++ if (hash_matches && bucket->nr == trace_nr && ++ memcmp(bucket->data, new_bucket->data, trace_len) == 0) { ++ pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); ++ return id; ++ } ++ if (bucket && !(flags & BPF_F_REUSE_STACKID)) { ++ pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); ++ return -EEXIST; ++ } ++ } else { ++ if (hash_matches && bucket->nr == trace_nr && ++ memcmp(bucket->data, ips, trace_len) == 0) ++ return id; ++ if (bucket && !(flags & BPF_F_REUSE_STACKID)) ++ return -EEXIST; ++ ++ new_bucket = (struct stack_map_bucket *) ++ pcpu_freelist_pop(&smap->freelist); ++ if (unlikely(!new_bucket)) ++ return -ENOMEM; ++ memcpy(new_bucket->data, ips, trace_len); ++ } ++ ++ new_bucket->hash = hash; ++ new_bucket->nr = trace_nr; ++ ++ old_bucket = xchg(&smap->buckets[id], new_bucket); ++ if (old_bucket) ++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); ++ return id; ++} ++ ++const struct bpf_func_proto bpf_get_stackid_proto = { ++ .func = bpf_get_stackid, ++ .gpl_only = true, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_CONST_MAP_PTR, ++ .arg3_type = ARG_ANYTHING, ++}; ++ ++BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, ++ u64, flags) ++{ ++ u32 init_nr, trace_nr, copy_len, elem_size, num_elem; ++ bool user_build_id = flags & BPF_F_USER_BUILD_ID; ++ u32 skip = flags & BPF_F_SKIP_FIELD_MASK; ++ bool user = flags & BPF_F_USER_STACK; ++ struct perf_callchain_entry *trace; ++ bool kernel = !user; ++ int err = -EINVAL; ++ u64 *ips; ++ ++ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | ++ BPF_F_USER_BUILD_ID))) ++ goto clear; ++ if (kernel && user_build_id) ++ goto clear; ++ ++ elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id) ++ : sizeof(u64); ++ if (unlikely(size % elem_size)) ++ goto clear; ++ ++ num_elem = size / elem_size; ++ if (sysctl_perf_event_max_stack < num_elem) ++ init_nr = 0; ++ else ++ init_nr = sysctl_perf_event_max_stack - num_elem; ++ trace = get_perf_callchain(regs, init_nr, kernel, user, ++ sysctl_perf_event_max_stack, false, false); ++ if (unlikely(!trace)) ++ goto err_fault; ++ ++ trace_nr = trace->nr - init_nr; ++ if (trace_nr < skip) ++ goto err_fault; ++ ++ trace_nr -= skip; ++ trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem; ++ copy_len = trace_nr * elem_size; ++ ips = trace->ip + skip + init_nr; ++ if (user && user_build_id) ++ stack_map_get_build_id_offset(buf, ips, trace_nr, user); ++ else ++ memcpy(buf, ips, copy_len); ++ ++ if (size > copy_len) ++ memset(buf + copy_len, 0, size - copy_len); ++ return copy_len; ++ ++err_fault: ++ err = -EFAULT; ++clear: ++ memset(buf, 0, size); ++ return err; ++} ++ ++const struct bpf_func_proto bpf_get_stack_proto = { ++ .func = bpf_get_stack, ++ .gpl_only = true, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg3_type = ARG_CONST_SIZE_OR_ZERO, ++ .arg4_type = ARG_ANYTHING, ++}; ++ ++/* Called from eBPF program */ ++static void *stack_map_lookup_elem(struct bpf_map *map, void *key) ++{ ++ return ERR_PTR(-EOPNOTSUPP); ++} ++ ++/* Called from syscall */ ++int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) ++{ ++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); ++ struct stack_map_bucket *bucket, *old_bucket; ++ u32 id = *(u32 *)key, trace_len; ++ ++ if (unlikely(id >= smap->n_buckets)) ++ return -ENOENT; ++ ++ bucket = xchg(&smap->buckets[id], NULL); ++ if (!bucket) ++ return -ENOENT; ++ ++ trace_len = bucket->nr * stack_map_data_size(map); ++ memcpy(value, bucket->data, trace_len); ++ memset(value + trace_len, 0, map->value_size - trace_len); ++ ++ old_bucket = xchg(&smap->buckets[id], bucket); ++ if (old_bucket) ++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); ++ return 0; ++} ++ ++static int stack_map_get_next_key(struct bpf_map *map, void *key, ++ void *next_key) ++{ ++ struct bpf_stack_map *smap = container_of(map, ++ struct bpf_stack_map, map); ++ u32 id; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ if (!key) { ++ id = 0; ++ } else { ++ id = *(u32 *)key; ++ if (id >= smap->n_buckets || !smap->buckets[id]) ++ id = 0; ++ else ++ id++; ++ } ++ ++ while (id < smap->n_buckets && !smap->buckets[id]) ++ id++; ++ ++ if (id >= smap->n_buckets) ++ return -ENOENT; ++ ++ *(u32 *)next_key = id; ++ return 0; ++} ++ ++static int stack_map_update_elem(struct bpf_map *map, void *key, void *value, ++ u64 map_flags) ++{ ++ return -EINVAL; ++} ++ ++/* Called from syscall or from eBPF program */ ++static int stack_map_delete_elem(struct bpf_map *map, void *key) ++{ ++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); ++ struct stack_map_bucket *old_bucket; ++ u32 id = *(u32 *)key; ++ ++ if (unlikely(id >= smap->n_buckets)) ++ return -E2BIG; ++ ++ old_bucket = xchg(&smap->buckets[id], NULL); ++ if (old_bucket) { ++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); ++ return 0; ++ } else { ++ return -ENOENT; ++ } ++} ++ ++/* Called when map->refcnt goes to zero, either from workqueue or from syscall */ ++static void stack_map_free(struct bpf_map *map) ++{ ++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); ++ ++ /* wait for bpf programs to complete before freeing stack map */ ++ synchronize_rcu(); ++ ++ bpf_map_area_free(smap->elems); ++ pcpu_freelist_destroy(&smap->freelist); ++ bpf_map_area_free(smap); ++ put_callchain_buffers(); ++} ++ ++const struct bpf_map_ops stack_trace_map_ops = { ++ .map_alloc = stack_map_alloc, ++ .map_free = stack_map_free, ++ .map_get_next_key = stack_map_get_next_key, ++ .map_lookup_elem = stack_map_lookup_elem, ++ .map_update_elem = stack_map_update_elem, ++ .map_delete_elem = stack_map_delete_elem, ++ .map_check_btf = map_check_no_btf, ++}; ++ ++static int __init stack_map_init(void) ++{ ++ int cpu; ++ struct stack_map_irq_work *work; ++ ++ for_each_possible_cpu(cpu) { ++ work = per_cpu_ptr(&up_read_work, cpu); ++ init_irq_work(&work->irq_work, do_up_read); ++ } ++ return 0; ++} ++subsys_initcall(stack_map_init); +--- a/kernel/bpf/syscall.c ++++ b/kernel/bpf/syscall.c +@@ -1,106 +1,333 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. +- * +- * This program is distributed in the hope that it will be useful, but +- * WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. + */ + #include ++#include ++#include + #include + #include ++#include ++#include + #include ++#include + #include ++#include + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++ ++#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ ++ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ ++ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ ++ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) ++#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) ++#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) ++ ++#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) ++ ++DEFINE_PER_CPU(int, bpf_prog_active); ++static DEFINE_IDR(prog_idr); ++static DEFINE_SPINLOCK(prog_idr_lock); ++static DEFINE_IDR(map_idr); ++static DEFINE_SPINLOCK(map_idr_lock); + + int sysctl_unprivileged_bpf_disabled __read_mostly; + +-static LIST_HEAD(bpf_map_types); ++static const struct bpf_map_ops * const bpf_map_types[] = { ++#define BPF_PROG_TYPE(_id, _ops) ++#define BPF_MAP_TYPE(_id, _ops) \ ++ [_id] = &_ops, ++#include ++#undef BPF_PROG_TYPE ++#undef BPF_MAP_TYPE ++}; ++ ++/* ++ * If we're handed a bigger struct than we know of, ensure all the unknown bits ++ * are 0 - i.e. new user-space does not rely on any kernel feature extensions ++ * we don't know about yet. ++ * ++ * There is a ToCToU between this function call and the following ++ * copy_from_user() call. However, this is not a concern since this function is ++ * meant to be a future-proofing of bits. ++ */ ++int bpf_check_uarg_tail_zero(void __user *uaddr, ++ size_t expected_size, ++ size_t actual_size) ++{ ++ unsigned char __user *addr; ++ unsigned char __user *end; ++ unsigned char val; ++ int err; ++ ++ if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ ++ return -E2BIG; ++ ++ if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) ++ return -EFAULT; ++ ++ if (actual_size <= expected_size) ++ return 0; ++ ++ addr = uaddr + expected_size; ++ end = uaddr + actual_size; ++ ++ for (; addr < end; addr++) { ++ err = get_user(val, addr); ++ if (err) ++ return err; ++ if (val) ++ return -E2BIG; ++ } ++ ++ return 0; ++} + + static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) + { +- struct bpf_map_type_list *tl; ++ const struct bpf_map_ops *ops; ++ u32 type = attr->map_type; + struct bpf_map *map; ++ int err; + +- list_for_each_entry(tl, &bpf_map_types, list_node) { +- if (tl->type == attr->map_type) { +- map = tl->ops->map_alloc(attr); +- if (IS_ERR(map)) +- return map; +- map->ops = tl->ops; +- map->map_type = attr->map_type; +- return map; +- } ++ if (type >= ARRAY_SIZE(bpf_map_types)) ++ return ERR_PTR(-EINVAL); ++ ops = bpf_map_types[type]; ++ if (!ops) ++ return ERR_PTR(-EINVAL); ++ ++ if (ops->map_alloc_check) { ++ err = ops->map_alloc_check(attr); ++ if (err) ++ return ERR_PTR(err); ++ } ++ map = ops->map_alloc(attr); ++ if (IS_ERR(map)) ++ return map; ++ map->ops = ops; ++ map->map_type = type; ++ return map; ++} ++ ++void *bpf_map_area_alloc(u64 size, int numa_node) ++{ ++ /* We really just want to fail instead of triggering OOM killer ++ * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, ++ * which is used for lower order allocation requests. ++ * ++ * It has been observed that higher order allocation requests done by ++ * vmalloc with __GFP_NORETRY being set might fail due to not trying ++ * to reclaim memory from the page cache, thus we set ++ * __GFP_RETRY_MAYFAIL to avoid such situations. ++ */ ++ ++ const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; ++ void *area; ++ ++ if (size >= SIZE_MAX) ++ return NULL; ++ ++ if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { ++ area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, ++ numa_node); ++ if (area != NULL) ++ return area; + } +- return ERR_PTR(-EINVAL); ++ ++ return __vmalloc_node_range(size, 1, ++ VMALLOC_START, VMALLOC_END, ++ GFP_KERNEL | flags, ++ PAGE_KERNEL, 0, numa_node, ++ __builtin_return_address(0)); + } + +-/* boot time registration of different map implementations */ +-void bpf_register_map_type(struct bpf_map_type_list *tl) ++void bpf_map_area_free(void *area) + { +- list_add(&tl->list_node, &bpf_map_types); ++ kvfree(area); + } + +-static int bpf_map_charge_memlock(struct bpf_map *map) ++static u32 bpf_map_flags_retain_permanent(u32 flags) + { +- struct user_struct *user = get_current_user(); +- unsigned long memlock_limit; ++ /* Some map creation flags are not tied to the map object but ++ * rather to the map fd instead, so they have no meaning upon ++ * map object inspection since multiple file descriptors with ++ * different (access) properties can exist here. Thus, given ++ * this has zero meaning for the map itself, lets clear these ++ * from here. ++ */ ++ return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); ++} ++ ++void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) ++{ ++ map->map_type = attr->map_type; ++ map->key_size = attr->key_size; ++ map->value_size = attr->value_size; ++ map->max_entries = attr->max_entries; ++ map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); ++ map->numa_node = bpf_map_attr_numa_node(attr); ++} ++ ++static int bpf_charge_memlock(struct user_struct *user, u32 pages) ++{ ++ unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; ++ ++ if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { ++ atomic_long_sub(pages, &user->locked_vm); ++ return -EPERM; ++ } ++ return 0; ++} ++ ++static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) ++{ ++ if (user) ++ atomic_long_sub(pages, &user->locked_vm); ++} + +- memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; ++int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size) ++{ ++ u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; ++ struct user_struct *user; ++ int ret; + +- atomic_long_add(map->pages, &user->locked_vm); ++ if (size >= U32_MAX - PAGE_SIZE) ++ return -E2BIG; + +- if (atomic_long_read(&user->locked_vm) > memlock_limit) { +- atomic_long_sub(map->pages, &user->locked_vm); ++ user = get_current_user(); ++ ret = bpf_charge_memlock(user, pages); ++ if (ret) { + free_uid(user); +- return -EPERM; ++ return ret; + } +- map->user = user; ++ ++ mem->pages = pages; ++ mem->user = user; ++ + return 0; + } + +-static void bpf_map_uncharge_memlock(struct bpf_map *map) ++void bpf_map_charge_finish(struct bpf_map_memory *mem) + { +- struct user_struct *user = map->user; ++ bpf_uncharge_memlock(mem->user, mem->pages); ++ free_uid(mem->user); ++} + +- atomic_long_sub(map->pages, &user->locked_vm); +- free_uid(user); ++void bpf_map_charge_move(struct bpf_map_memory *dst, ++ struct bpf_map_memory *src) ++{ ++ *dst = *src; ++ ++ /* Make sure src will not be used for the redundant uncharging. */ ++ memset(src, 0, sizeof(struct bpf_map_memory)); ++} ++ ++int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) ++{ ++ int ret; ++ ++ ret = bpf_charge_memlock(map->memory.user, pages); ++ if (ret) ++ return ret; ++ map->memory.pages += pages; ++ return ret; ++} ++ ++void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) ++{ ++ bpf_uncharge_memlock(map->memory.user, pages); ++ map->memory.pages -= pages; ++} ++ ++static int bpf_map_alloc_id(struct bpf_map *map) ++{ ++ int id; ++ ++ idr_preload(GFP_KERNEL); ++ spin_lock_bh(&map_idr_lock); ++ id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); ++ if (id > 0) ++ map->id = id; ++ spin_unlock_bh(&map_idr_lock); ++ idr_preload_end(); ++ ++ if (WARN_ON_ONCE(!id)) ++ return -ENOSPC; ++ ++ return id > 0 ? 0 : id; ++} ++ ++void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) ++{ ++ unsigned long flags; ++ ++ /* Offloaded maps are removed from the IDR store when their device ++ * disappears - even if someone holds an fd to them they are unusable, ++ * the memory is gone, all ops will fail; they are simply waiting for ++ * refcnt to drop to be freed. ++ */ ++ if (!map->id) ++ return; ++ ++ if (do_idr_lock) ++ spin_lock_irqsave(&map_idr_lock, flags); ++ else ++ __acquire(&map_idr_lock); ++ ++ idr_remove(&map_idr, map->id); ++ map->id = 0; ++ ++ if (do_idr_lock) ++ spin_unlock_irqrestore(&map_idr_lock, flags); ++ else ++ __release(&map_idr_lock); + } + + /* called from workqueue */ + static void bpf_map_free_deferred(struct work_struct *work) + { + struct bpf_map *map = container_of(work, struct bpf_map, work); ++ struct bpf_map_memory mem; + +- bpf_map_uncharge_memlock(map); ++ bpf_map_charge_move(&mem, &map->memory); + /* implementation dependent freeing */ + map->ops->map_free(map); ++ bpf_map_charge_finish(&mem); + } + + static void bpf_map_put_uref(struct bpf_map *map) + { + if (atomic_dec_and_test(&map->usercnt)) { +- if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) +- bpf_fd_array_map_clear(map); ++ if (map->ops->map_release_uref) ++ map->ops->map_release_uref(map); + } + } + + /* decrement map refcnt and schedule it for freeing via workqueue + * (unrelying map implementation ops->map_free() might sleep) + */ +-void bpf_map_put(struct bpf_map *map) ++static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) + { + if (atomic_dec_and_test(&map->refcnt)) { ++ /* bpf_map_free_id() must be called first */ ++ bpf_map_free_id(map, do_idr_lock); ++ btf_put(map->btf); + INIT_WORK(&map->work, bpf_map_free_deferred); + schedule_work(&map->work); + } + } + ++void bpf_map_put(struct bpf_map *map) ++{ ++ __bpf_map_put(map, true); ++} ++EXPORT_SYMBOL_GPL(bpf_map_put); ++ + void bpf_map_put_with_uref(struct bpf_map *map) + { + bpf_map_put_uref(map); +@@ -109,18 +336,110 @@ void bpf_map_put_with_uref(struct bpf_ma + + static int bpf_map_release(struct inode *inode, struct file *filp) + { +- bpf_map_put_with_uref(filp->private_data); ++ struct bpf_map *map = filp->private_data; ++ ++ if (map->ops->map_release) ++ map->ops->map_release(map, filp); ++ ++ bpf_map_put_with_uref(map); + return 0; + } + +-static const struct file_operations bpf_map_fops = { +- .release = bpf_map_release, ++static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) ++{ ++ fmode_t mode = f.file->f_mode; ++ ++ /* Our file permissions may have been overridden by global ++ * map permissions facing syscall side. ++ */ ++ if (READ_ONCE(map->frozen)) ++ mode &= ~FMODE_CAN_WRITE; ++ return mode; ++} ++ ++#ifdef CONFIG_PROC_FS ++static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) ++{ ++ const struct bpf_map *map = filp->private_data; ++ const struct bpf_array *array; ++ u32 owner_prog_type = 0; ++ u32 owner_jited = 0; ++ ++ if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { ++ array = container_of(map, struct bpf_array, map); ++ owner_prog_type = array->owner_prog_type; ++ owner_jited = array->owner_jited; ++ } ++ ++ seq_printf(m, ++ "map_type:\t%u\n" ++ "key_size:\t%u\n" ++ "value_size:\t%u\n" ++ "max_entries:\t%u\n" ++ "map_flags:\t%#x\n" ++ "memlock:\t%llu\n" ++ "map_id:\t%u\n" ++ "frozen:\t%u\n", ++ map->map_type, ++ map->key_size, ++ map->value_size, ++ map->max_entries, ++ map->map_flags, ++ map->memory.pages * 1ULL << PAGE_SHIFT, ++ map->id, ++ READ_ONCE(map->frozen)); ++ ++ if (owner_prog_type) { ++ seq_printf(m, "owner_prog_type:\t%u\n", ++ owner_prog_type); ++ seq_printf(m, "owner_jited:\t%u\n", ++ owner_jited); ++ } ++} ++#endif ++ ++static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, ++ loff_t *ppos) ++{ ++ /* We need this handler such that alloc_file() enables ++ * f_mode with FMODE_CAN_READ. ++ */ ++ return -EINVAL; ++} ++ ++static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, ++ size_t siz, loff_t *ppos) ++{ ++ /* We need this handler such that alloc_file() enables ++ * f_mode with FMODE_CAN_WRITE. ++ */ ++ return -EINVAL; ++} ++ ++const struct file_operations bpf_map_fops = { ++#ifdef CONFIG_PROC_FS ++ .show_fdinfo = bpf_map_show_fdinfo, ++#endif ++ .release = bpf_map_release, ++ .read = bpf_dummy_read, ++ .write = bpf_dummy_write, + }; + +-int bpf_map_new_fd(struct bpf_map *map) ++int bpf_map_new_fd(struct bpf_map *map, int flags) + { + return anon_inode_getfd("bpf-map", &bpf_map_fops, map, +- O_RDWR | O_CLOEXEC); ++ flags | O_CLOEXEC); ++} ++ ++int bpf_get_file_flag(int flags) ++{ ++ if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) ++ return -EINVAL; ++ if (flags & BPF_F_RDONLY) ++ return O_RDONLY; ++ if (flags & BPF_F_WRONLY) ++ return O_WRONLY; ++ return O_RDWR; + } + + /* helper macro to check that unused fields 'union bpf_attr' are zero */ +@@ -131,38 +450,171 @@ int bpf_map_new_fd(struct bpf_map *map) + offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ + sizeof(attr->CMD##_LAST_FIELD)) != NULL + +-#define BPF_MAP_CREATE_LAST_FIELD max_entries ++/* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. ++ * Return 0 on success and < 0 on error. ++ */ ++static int bpf_obj_name_cpy(char *dst, const char *src) ++{ ++ const char *end = src + BPF_OBJ_NAME_LEN; ++ ++ memset(dst, 0, BPF_OBJ_NAME_LEN); ++ /* Copy all isalnum(), '_' and '.' chars. */ ++ while (src < end && *src) { ++ if (!isalnum(*src) && ++ *src != '_' && *src != '.') ++ return -EINVAL; ++ *dst++ = *src++; ++ } ++ ++ /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ ++ if (src == end) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++int map_check_no_btf(const struct bpf_map *map, ++ const struct btf *btf, ++ const struct btf_type *key_type, ++ const struct btf_type *value_type) ++{ ++ return -ENOTSUPP; ++} ++ ++static int map_check_btf(struct bpf_map *map, const struct btf *btf, ++ u32 btf_key_id, u32 btf_value_id) ++{ ++ const struct btf_type *key_type, *value_type; ++ u32 key_size, value_size; ++ int ret = 0; ++ ++ /* Some maps allow key to be unspecified. */ ++ if (btf_key_id) { ++ key_type = btf_type_id_size(btf, &btf_key_id, &key_size); ++ if (!key_type || key_size != map->key_size) ++ return -EINVAL; ++ } else { ++ key_type = btf_type_by_id(btf, 0); ++ if (!map->ops->map_check_btf) ++ return -EINVAL; ++ } ++ ++ value_type = btf_type_id_size(btf, &btf_value_id, &value_size); ++ if (!value_type || value_size != map->value_size) ++ return -EINVAL; ++ ++ map->spin_lock_off = btf_find_spin_lock(btf, value_type); ++ ++ if (map_value_has_spin_lock(map)) { ++ if (map->map_flags & BPF_F_RDONLY_PROG) ++ return -EACCES; ++ if (map->map_type != BPF_MAP_TYPE_HASH && ++ map->map_type != BPF_MAP_TYPE_ARRAY && ++ map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && ++ map->map_type != BPF_MAP_TYPE_SK_STORAGE) ++ return -ENOTSUPP; ++ if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > ++ map->value_size) { ++ WARN_ONCE(1, ++ "verifier bug spin_lock_off %d value_size %d\n", ++ map->spin_lock_off, map->value_size); ++ return -EFAULT; ++ } ++ } ++ ++ if (map->ops->map_check_btf) ++ ret = map->ops->map_check_btf(map, btf, key_type, value_type); ++ ++ return ret; ++} ++ ++#define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id + /* called via syscall */ + static int map_create(union bpf_attr *attr) + { ++ int numa_node = bpf_map_attr_numa_node(attr); ++ struct bpf_map_memory mem; + struct bpf_map *map; ++ int f_flags; + int err; + + err = CHECK_ATTR(BPF_MAP_CREATE); + if (err) + return -EINVAL; + ++ f_flags = bpf_get_file_flag(attr->map_flags); ++ if (f_flags < 0) ++ return f_flags; ++ ++ if (numa_node != NUMA_NO_NODE && ++ ((unsigned int)numa_node >= nr_node_ids || ++ !node_online(numa_node))) ++ return -EINVAL; ++ + /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ + map = find_and_alloc_map(attr); + if (IS_ERR(map)) + return PTR_ERR(map); + ++ err = bpf_obj_name_cpy(map->name, attr->map_name); ++ if (err) ++ goto free_map; ++ + atomic_set(&map->refcnt, 1); + atomic_set(&map->usercnt, 1); + +- err = bpf_map_charge_memlock(map); ++ if (attr->btf_key_type_id || attr->btf_value_type_id) { ++ struct btf *btf; ++ ++ if (!attr->btf_value_type_id) { ++ err = -EINVAL; ++ goto free_map; ++ } ++ ++ btf = btf_get_by_fd(attr->btf_fd); ++ if (IS_ERR(btf)) { ++ err = PTR_ERR(btf); ++ goto free_map; ++ } ++ ++ err = map_check_btf(map, btf, attr->btf_key_type_id, ++ attr->btf_value_type_id); ++ if (err) { ++ btf_put(btf); ++ goto free_map; ++ } ++ ++ map->btf = btf; ++ map->btf_key_type_id = attr->btf_key_type_id; ++ map->btf_value_type_id = attr->btf_value_type_id; ++ } else { ++ map->spin_lock_off = -EINVAL; ++ } ++ ++ err = bpf_map_alloc_id(map); + if (err) +- goto free_map; ++ goto free_map_sec; + +- err = bpf_map_new_fd(map); +- if (err < 0) +- /* failed to allocate fd */ +- goto free_map; ++ err = bpf_map_new_fd(map, f_flags); ++ if (err < 0) { ++ /* failed to allocate fd. ++ * bpf_map_put_with_uref() is needed because the above ++ * bpf_map_alloc_id() has published the map ++ * to the userspace and the userspace may ++ * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. ++ */ ++ bpf_map_put_with_uref(map); ++ return err; ++ } + + return err; + ++free_map_sec: + free_map: ++ btf_put(map->btf); ++ bpf_map_charge_move(&mem, &map->memory); + map->ops->map_free(map); ++ bpf_map_charge_finish(&mem); + return err; + } + +@@ -194,6 +646,7 @@ struct bpf_map *bpf_map_inc(struct bpf_m + atomic_inc(&map->usercnt); + return map; + } ++EXPORT_SYMBOL_GPL(bpf_map_inc); + + struct bpf_map *bpf_map_get_with_uref(u32 ufd) + { +@@ -210,59 +663,155 @@ struct bpf_map *bpf_map_get_with_uref(u3 + return map; + } + +-/* helper to convert user pointers passed inside __aligned_u64 fields */ +-static void __user *u64_to_ptr(__u64 val) ++/* map_idr_lock should have been held */ ++static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, ++ bool uref) ++{ ++ int refold; ++ ++ refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); ++ ++ if (refold >= BPF_MAX_REFCNT) { ++ __bpf_map_put(map, false); ++ return ERR_PTR(-EBUSY); ++ } ++ ++ if (!refold) ++ return ERR_PTR(-ENOENT); ++ ++ if (uref) ++ atomic_inc(&map->usercnt); ++ ++ return map; ++} ++ ++struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, bool uref) ++{ ++ spin_lock_bh(&map_idr_lock); ++ map = __bpf_map_inc_not_zero(map, uref); ++ spin_unlock_bh(&map_idr_lock); ++ ++ return map; ++} ++EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); ++ ++int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) ++{ ++ return -ENOTSUPP; ++} ++ ++static void *__bpf_copy_key(void __user *ukey, u64 key_size) + { +- return (void __user *) (unsigned long) val; ++ if (key_size) ++ return memdup_user(ukey, key_size); ++ ++ if (ukey) ++ return ERR_PTR(-EINVAL); ++ ++ return NULL; + } + + /* last field in 'union bpf_attr' used by this command */ +-#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value ++#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags + + static int map_lookup_elem(union bpf_attr *attr) + { +- void __user *ukey = u64_to_ptr(attr->key); +- void __user *uvalue = u64_to_ptr(attr->value); ++ void __user *ukey = u64_to_user_ptr(attr->key); ++ void __user *uvalue = u64_to_user_ptr(attr->value); + int ufd = attr->map_fd; + struct bpf_map *map; + void *key, *value, *ptr; ++ u32 value_size; + struct fd f; + int err; + + if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) + return -EINVAL; + ++ if (attr->flags & ~BPF_F_LOCK) ++ return -EINVAL; ++ + f = fdget(ufd); + map = __bpf_map_get(f); + if (IS_ERR(map)) + return PTR_ERR(map); ++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { ++ err = -EPERM; ++ goto err_put; ++ } + +- err = -ENOMEM; +- key = kmalloc(map->key_size, GFP_USER); +- if (!key) ++ if ((attr->flags & BPF_F_LOCK) && ++ !map_value_has_spin_lock(map)) { ++ err = -EINVAL; + goto err_put; ++ } + +- err = -EFAULT; +- if (copy_from_user(key, ukey, map->key_size) != 0) +- goto free_key; ++ key = __bpf_copy_key(ukey, map->key_size); ++ if (IS_ERR(key)) { ++ err = PTR_ERR(key); ++ goto err_put; ++ } ++ ++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) ++ value_size = round_up(map->value_size, 8) * num_possible_cpus(); ++ else if (IS_FD_MAP(map)) ++ value_size = sizeof(u32); ++ else ++ value_size = map->value_size; + + err = -ENOMEM; +- value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN); ++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); + if (!value) + goto free_key; + +- rcu_read_lock(); +- ptr = map->ops->map_lookup_elem(map, key); +- if (ptr) +- memcpy(value, ptr, map->value_size); +- rcu_read_unlock(); ++ preempt_disable(); ++ this_cpu_inc(bpf_prog_active); ++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { ++ err = bpf_percpu_hash_copy(map, key, value); ++ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { ++ err = bpf_percpu_array_copy(map, key, value); ++ } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { ++ err = bpf_stackmap_copy(map, key, value); ++ } else if (IS_FD_ARRAY(map)) { ++ err = bpf_fd_array_map_lookup_elem(map, key, value); ++ } else if (IS_FD_HASH(map)) { ++ err = bpf_fd_htab_map_lookup_elem(map, key, value); ++ } else if (map->map_type == BPF_MAP_TYPE_QUEUE || ++ map->map_type == BPF_MAP_TYPE_STACK) { ++ err = map->ops->map_peek_elem(map, value); ++ } else { ++ rcu_read_lock(); ++ if (map->ops->map_lookup_elem_sys_only) ++ ptr = map->ops->map_lookup_elem_sys_only(map, key); ++ else ++ ptr = map->ops->map_lookup_elem(map, key); ++ if (IS_ERR(ptr)) { ++ err = PTR_ERR(ptr); ++ } else if (!ptr) { ++ err = -ENOENT; ++ } else { ++ err = 0; ++ if (attr->flags & BPF_F_LOCK) ++ /* lock 'ptr' and copy everything but lock */ ++ copy_map_value_locked(map, value, ptr, true); ++ else ++ copy_map_value(map, value, ptr); ++ /* mask lock, since value wasn't zero inited */ ++ check_and_init_map_lock(map, value); ++ } ++ rcu_read_unlock(); ++ } ++ this_cpu_dec(bpf_prog_active); ++ preempt_enable(); + +- err = -ENOENT; +- if (!ptr) ++ if (err) + goto free_value; + + err = -EFAULT; +- if (copy_to_user(uvalue, value, map->value_size) != 0) ++ if (copy_to_user(uvalue, value, value_size) != 0) + goto free_value; + + err = 0; +@@ -276,15 +825,27 @@ err_put: + return err; + } + ++static void maybe_wait_bpf_programs(struct bpf_map *map) ++{ ++ /* Wait for any running BPF programs to complete so that ++ * userspace, when we return to it, knows that all programs ++ * that could be running use the new map value. ++ */ ++ if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || ++ map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) ++ synchronize_rcu(); ++} ++ + #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags + + static int map_update_elem(union bpf_attr *attr) + { +- void __user *ukey = u64_to_ptr(attr->key); +- void __user *uvalue = u64_to_ptr(attr->value); ++ void __user *ukey = u64_to_user_ptr(attr->key); ++ void __user *uvalue = u64_to_user_ptr(attr->value); + int ufd = attr->map_fd; + struct bpf_map *map; + void *key, *value; ++ u32 value_size; + struct fd f; + int err; + +@@ -295,32 +856,79 @@ static int map_update_elem(union bpf_att + map = __bpf_map_get(f); + if (IS_ERR(map)) + return PTR_ERR(map); ++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { ++ err = -EPERM; ++ goto err_put; ++ } + +- err = -ENOMEM; +- key = kmalloc(map->key_size, GFP_USER); +- if (!key) ++ if ((attr->flags & BPF_F_LOCK) && ++ !map_value_has_spin_lock(map)) { ++ err = -EINVAL; ++ goto err_put; ++ } ++ ++ key = __bpf_copy_key(ukey, map->key_size); ++ if (IS_ERR(key)) { ++ err = PTR_ERR(key); + goto err_put; ++ } + +- err = -EFAULT; +- if (copy_from_user(key, ukey, map->key_size) != 0) +- goto free_key; ++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) ++ value_size = round_up(map->value_size, 8) * num_possible_cpus(); ++ else ++ value_size = map->value_size; + + err = -ENOMEM; +- value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN); ++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); + if (!value) + goto free_key; + + err = -EFAULT; +- if (copy_from_user(value, uvalue, map->value_size) != 0) ++ if (copy_from_user(value, uvalue, value_size) != 0) + goto free_value; + +- /* eBPF program that use maps are running under rcu_read_lock(), +- * therefore all map accessors rely on this fact, so do the same here +- */ +- rcu_read_lock(); +- err = map->ops->map_update_elem(map, key, value, attr->flags); +- rcu_read_unlock(); ++ /* Need to create a kthread, thus must support schedule */ ++ if (map->map_type == BPF_MAP_TYPE_CPUMAP || ++ map->map_type == BPF_MAP_TYPE_SOCKHASH || ++ map->map_type == BPF_MAP_TYPE_SOCKMAP) { ++ err = map->ops->map_update_elem(map, key, value, attr->flags); ++ goto out; ++ } + ++ /* must increment bpf_prog_active to avoid kprobe+bpf triggering from ++ * inside bpf map update or delete otherwise deadlocks are possible ++ */ ++ preempt_disable(); ++ __this_cpu_inc(bpf_prog_active); ++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || ++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { ++ err = bpf_percpu_hash_update(map, key, value, attr->flags); ++ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { ++ err = bpf_percpu_array_update(map, key, value, attr->flags); ++ } else if (IS_FD_ARRAY(map)) { ++ rcu_read_lock(); ++ err = bpf_fd_array_map_update_elem(map, f.file, key, value, ++ attr->flags); ++ rcu_read_unlock(); ++ } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { ++ rcu_read_lock(); ++ err = bpf_fd_htab_map_update_elem(map, f.file, key, value, ++ attr->flags); ++ rcu_read_unlock(); ++ } else if (map->map_type == BPF_MAP_TYPE_QUEUE || ++ map->map_type == BPF_MAP_TYPE_STACK) { ++ err = map->ops->map_push_elem(map, value, attr->flags); ++ } else { ++ rcu_read_lock(); ++ err = map->ops->map_update_elem(map, key, value, attr->flags); ++ rcu_read_unlock(); ++ } ++ __this_cpu_dec(bpf_prog_active); ++ preempt_enable(); ++ maybe_wait_bpf_programs(map); ++out: + free_value: + kfree(value); + free_key: +@@ -334,7 +942,7 @@ err_put: + + static int map_delete_elem(union bpf_attr *attr) + { +- void __user *ukey = u64_to_ptr(attr->key); ++ void __user *ukey = u64_to_user_ptr(attr->key); + int ufd = attr->map_fd; + struct bpf_map *map; + struct fd f; +@@ -348,21 +956,25 @@ static int map_delete_elem(union bpf_att + map = __bpf_map_get(f); + if (IS_ERR(map)) + return PTR_ERR(map); +- +- err = -ENOMEM; +- key = kmalloc(map->key_size, GFP_USER); +- if (!key) ++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { ++ err = -EPERM; + goto err_put; ++ } + +- err = -EFAULT; +- if (copy_from_user(key, ukey, map->key_size) != 0) +- goto free_key; ++ key = __bpf_copy_key(ukey, map->key_size); ++ if (IS_ERR(key)) { ++ err = PTR_ERR(key); ++ goto err_put; ++ } + ++ preempt_disable(); ++ __this_cpu_inc(bpf_prog_active); + rcu_read_lock(); + err = map->ops->map_delete_elem(map, key); + rcu_read_unlock(); +- +-free_key: ++ __this_cpu_dec(bpf_prog_active); ++ preempt_enable(); ++ maybe_wait_bpf_programs(map); + kfree(key); + err_put: + fdput(f); +@@ -374,8 +986,8 @@ err_put: + + static int map_get_next_key(union bpf_attr *attr) + { +- void __user *ukey = u64_to_ptr(attr->key); +- void __user *unext_key = u64_to_ptr(attr->next_key); ++ void __user *ukey = u64_to_user_ptr(attr->key); ++ void __user *unext_key = u64_to_user_ptr(attr->next_key); + int ufd = attr->map_fd; + struct bpf_map *map; + void *key, *next_key; +@@ -389,15 +1001,20 @@ static int map_get_next_key(union bpf_at + map = __bpf_map_get(f); + if (IS_ERR(map)) + return PTR_ERR(map); +- +- err = -ENOMEM; +- key = kmalloc(map->key_size, GFP_USER); +- if (!key) ++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { ++ err = -EPERM; + goto err_put; ++ } + +- err = -EFAULT; +- if (copy_from_user(key, ukey, map->key_size) != 0) +- goto free_key; ++ if (ukey) { ++ key = __bpf_copy_key(ukey, map->key_size); ++ if (IS_ERR(key)) { ++ err = PTR_ERR(key); ++ goto err_put; ++ } ++ } else { ++ key = NULL; ++ } + + err = -ENOMEM; + next_key = kmalloc(map->key_size, GFP_USER); +@@ -425,77 +1042,126 @@ err_put: + return err; + } + +-static LIST_HEAD(bpf_prog_types); ++#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value + +-static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) ++static int map_lookup_and_delete_elem(union bpf_attr *attr) + { +- struct bpf_prog_type_list *tl; ++ void __user *ukey = u64_to_user_ptr(attr->key); ++ void __user *uvalue = u64_to_user_ptr(attr->value); ++ int ufd = attr->map_fd; ++ struct bpf_map *map; ++ void *key, *value; ++ u32 value_size; ++ struct fd f; ++ int err; + +- list_for_each_entry(tl, &bpf_prog_types, list_node) { +- if (tl->type == type) { +- prog->aux->ops = tl->ops; +- prog->type = type; +- return 0; +- } ++ if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) ++ return -EINVAL; ++ ++ f = fdget(ufd); ++ map = __bpf_map_get(f); ++ if (IS_ERR(map)) ++ return PTR_ERR(map); ++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || ++ !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { ++ err = -EPERM; ++ goto err_put; + } + +- return -EINVAL; +-} ++ key = __bpf_copy_key(ukey, map->key_size); ++ if (IS_ERR(key)) { ++ err = PTR_ERR(key); ++ goto err_put; ++ } + +-void bpf_register_prog_type(struct bpf_prog_type_list *tl) +-{ +- list_add(&tl->list_node, &bpf_prog_types); ++ value_size = map->value_size; ++ ++ err = -ENOMEM; ++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); ++ if (!value) ++ goto free_key; ++ ++ if (map->map_type == BPF_MAP_TYPE_QUEUE || ++ map->map_type == BPF_MAP_TYPE_STACK) { ++ err = map->ops->map_pop_elem(map, value); ++ } else { ++ err = -ENOTSUPP; ++ } ++ ++ if (err) ++ goto free_value; ++ ++ if (copy_to_user(uvalue, value, value_size) != 0) { ++ err = -EFAULT; ++ goto free_value; ++ } ++ ++ err = 0; ++ ++free_value: ++ kfree(value); ++free_key: ++ kfree(key); ++err_put: ++ fdput(f); ++ return err; + } + +-/* fixup insn->imm field of bpf_call instructions: +- * if (insn->imm == BPF_FUNC_map_lookup_elem) +- * insn->imm = bpf_map_lookup_elem - __bpf_call_base; +- * else if (insn->imm == BPF_FUNC_map_update_elem) +- * insn->imm = bpf_map_update_elem - __bpf_call_base; +- * else ... +- * +- * this function is called after eBPF program passed verification +- */ +-static void fixup_bpf_calls(struct bpf_prog *prog) ++#define BPF_MAP_FREEZE_LAST_FIELD map_fd ++ ++static int map_freeze(const union bpf_attr *attr) + { +- const struct bpf_func_proto *fn; +- int i; ++ int err = 0, ufd = attr->map_fd; ++ struct bpf_map *map; ++ struct fd f; + +- for (i = 0; i < prog->len; i++) { +- struct bpf_insn *insn = &prog->insnsi[i]; ++ if (CHECK_ATTR(BPF_MAP_FREEZE)) ++ return -EINVAL; + +- if (insn->code == (BPF_JMP | BPF_CALL)) { +- /* we reach here when program has bpf_call instructions +- * and it passed bpf_check(), means that +- * ops->get_func_proto must have been supplied, check it +- */ +- BUG_ON(!prog->aux->ops->get_func_proto); ++ f = fdget(ufd); ++ map = __bpf_map_get(f); ++ if (IS_ERR(map)) ++ return PTR_ERR(map); ++ if (READ_ONCE(map->frozen)) { ++ err = -EBUSY; ++ goto err_put; ++ } ++ if (!capable(CAP_SYS_ADMIN)) { ++ err = -EPERM; ++ goto err_put; ++ } + +- if (insn->imm == BPF_FUNC_get_route_realm) +- prog->dst_needed = 1; +- if (insn->imm == BPF_FUNC_get_prandom_u32) +- bpf_user_rnd_init_once(); +- if (insn->imm == BPF_FUNC_tail_call) { +- /* mark bpf_tail_call as different opcode +- * to avoid conditional branch in +- * interpeter for every normal call +- * and to prevent accidental JITing by +- * JIT compiler that doesn't support +- * bpf_tail_call yet +- */ +- insn->imm = 0; +- insn->code |= BPF_X; +- continue; +- } ++ WRITE_ONCE(map->frozen, true); ++err_put: ++ fdput(f); ++ return err; ++} + +- fn = prog->aux->ops->get_func_proto(insn->imm); +- /* all functions that have prototype and verifier allowed +- * programs to call them, must be real in-kernel functions +- */ +- BUG_ON(!fn->func); +- insn->imm = fn->func - __bpf_call_base; +- } +- } ++static const struct bpf_prog_ops * const bpf_prog_types[] = { ++#define BPF_PROG_TYPE(_id, _name) \ ++ [_id] = & _name ## _prog_ops, ++#define BPF_MAP_TYPE(_id, _ops) ++#include ++#undef BPF_PROG_TYPE ++#undef BPF_MAP_TYPE ++}; ++ ++static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) ++{ ++ const struct bpf_prog_ops *ops; ++ ++ if (type >= ARRAY_SIZE(bpf_prog_types)) ++ return -EINVAL; ++ ops = bpf_prog_types[type]; ++ if (!ops) ++ return -EINVAL; ++ ++ if (!bpf_prog_is_dev_bound(prog->aux)) ++ prog->aux->ops = ops; ++ else ++ return -EINVAL; ++ prog->type = type; ++ return 0; + } + + /* drop refcnt on maps used by eBPF program and free auxilary data */ +@@ -509,19 +1175,39 @@ static void free_used_maps(struct bpf_pr + kfree(aux->used_maps); + } + ++int __bpf_prog_charge(struct user_struct *user, u32 pages) ++{ ++ unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; ++ unsigned long user_bufs; ++ ++ if (user) { ++ user_bufs = atomic_long_add_return(pages, &user->locked_vm); ++ if (user_bufs > memlock_limit) { ++ atomic_long_sub(pages, &user->locked_vm); ++ return -EPERM; ++ } ++ } ++ ++ return 0; ++} ++ ++void __bpf_prog_uncharge(struct user_struct *user, u32 pages) ++{ ++ if (user) ++ atomic_long_sub(pages, &user->locked_vm); ++} ++ + static int bpf_prog_charge_memlock(struct bpf_prog *prog) + { + struct user_struct *user = get_current_user(); +- unsigned long memlock_limit; ++ int ret; + +- memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; +- +- atomic_long_add(prog->pages, &user->locked_vm); +- if (atomic_long_read(&user->locked_vm) > memlock_limit) { +- atomic_long_sub(prog->pages, &user->locked_vm); ++ ret = __bpf_prog_charge(user, prog->pages); ++ if (ret) { + free_uid(user); +- return -EPERM; ++ return ret; + } ++ + prog->aux->user = user; + return 0; + } +@@ -530,30 +1216,87 @@ static void bpf_prog_uncharge_memlock(st + { + struct user_struct *user = prog->aux->user; + +- atomic_long_sub(prog->pages, &user->locked_vm); ++ __bpf_prog_uncharge(user, prog->pages); + free_uid(user); + } + +-static void __prog_put_common(struct rcu_head *rcu) ++static int bpf_prog_alloc_id(struct bpf_prog *prog) ++{ ++ int id; ++ ++ idr_preload(GFP_KERNEL); ++ spin_lock_bh(&prog_idr_lock); ++ id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); ++ if (id > 0) ++ prog->aux->id = id; ++ spin_unlock_bh(&prog_idr_lock); ++ idr_preload_end(); ++ ++ /* id is in [1, INT_MAX) */ ++ if (WARN_ON_ONCE(!id)) ++ return -ENOSPC; ++ ++ return id > 0 ? 0 : id; ++} ++ ++void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) ++{ ++ /* cBPF to eBPF migrations are currently not in the idr store. ++ * Offloaded programs are removed from the store when their device ++ * disappears - even if someone grabs an fd to them they are unusable, ++ * simply waiting for refcnt to drop to be freed. ++ */ ++ if (!prog->aux->id) ++ return; ++ ++ if (do_idr_lock) ++ spin_lock_bh(&prog_idr_lock); ++ else ++ __acquire(&prog_idr_lock); ++ ++ idr_remove(&prog_idr, prog->aux->id); ++ prog->aux->id = 0; ++ ++ if (do_idr_lock) ++ spin_unlock_bh(&prog_idr_lock); ++ else ++ __release(&prog_idr_lock); ++} ++ ++static void __bpf_prog_put_rcu(struct rcu_head *rcu) + { + struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + ++ kvfree(aux->func_info); + free_used_maps(aux); + bpf_prog_uncharge_memlock(aux->prog); + bpf_prog_free(aux->prog); + } + +-/* version of bpf_prog_put() that is called after a grace period */ +-void bpf_prog_put_rcu(struct bpf_prog *prog) ++static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) ++{ ++ bpf_prog_kallsyms_del_all(prog); ++ btf_put(prog->aux->btf); ++ bpf_prog_free_linfo(prog); ++ ++ if (deferred) ++ call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); ++ else ++ __bpf_prog_put_rcu(&prog->aux->rcu); ++} ++ ++static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) + { +- if (atomic_dec_and_test(&prog->aux->refcnt)) +- call_rcu(&prog->aux->rcu, __prog_put_common); ++ if (atomic_dec_and_test(&prog->aux->refcnt)) { ++ /* bpf_prog_free_id() must be called first */ ++ bpf_prog_free_id(prog, do_idr_lock); ++ __bpf_prog_put_noref(prog, true); ++ } + } + + void bpf_prog_put(struct bpf_prog *prog) + { +- if (atomic_dec_and_test(&prog->aux->refcnt)) +- __prog_put_common(&prog->aux->rcu); ++ __bpf_prog_put(prog, true); + } + EXPORT_SYMBOL_GPL(bpf_prog_put); + +@@ -561,12 +1304,68 @@ static int bpf_prog_release(struct inode + { + struct bpf_prog *prog = filp->private_data; + +- bpf_prog_put_rcu(prog); ++ bpf_prog_put(prog); + return 0; + } + +-static const struct file_operations bpf_prog_fops = { +- .release = bpf_prog_release, ++static void bpf_prog_get_stats(const struct bpf_prog *prog, ++ struct bpf_prog_stats *stats) ++{ ++ u64 nsecs = 0, cnt = 0; ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ const struct bpf_prog_stats *st; ++ unsigned int start; ++ u64 tnsecs, tcnt; ++ ++ st = per_cpu_ptr(prog->aux->stats, cpu); ++ do { ++ start = u64_stats_fetch_begin_irq(&st->syncp); ++ tnsecs = st->nsecs; ++ tcnt = st->cnt; ++ } while (u64_stats_fetch_retry_irq(&st->syncp, start)); ++ nsecs += tnsecs; ++ cnt += tcnt; ++ } ++ stats->nsecs = nsecs; ++ stats->cnt = cnt; ++} ++ ++#ifdef CONFIG_PROC_FS ++static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) ++{ ++ const struct bpf_prog *prog = filp->private_data; ++ char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; ++ struct bpf_prog_stats stats; ++ ++ bpf_prog_get_stats(prog, &stats); ++ bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); ++ seq_printf(m, ++ "prog_type:\t%u\n" ++ "prog_jited:\t%u\n" ++ "prog_tag:\t%s\n" ++ "memlock:\t%llu\n" ++ "prog_id:\t%u\n" ++ "run_time_ns:\t%llu\n" ++ "run_cnt:\t%llu\n", ++ prog->type, ++ prog->jited, ++ prog_tag, ++ prog->pages * 1ULL << PAGE_SHIFT, ++ prog->aux->id, ++ stats.nsecs, ++ stats.cnt); ++} ++#endif ++ ++const struct file_operations bpf_prog_fops = { ++#ifdef CONFIG_PROC_FS ++ .show_fdinfo = bpf_prog_show_fdinfo, ++#endif ++ .release = bpf_prog_release, ++ .read = bpf_dummy_read, ++ .write = bpf_dummy_write, + }; + + int bpf_prog_new_fd(struct bpf_prog *prog) +@@ -575,7 +1374,7 @@ int bpf_prog_new_fd(struct bpf_prog *pro + O_RDWR | O_CLOEXEC); + } + +-static struct bpf_prog *__bpf_prog_get(struct fd f) ++static struct bpf_prog *____bpf_prog_get(struct fd f) + { + if (!f.file) + return ERR_PTR(-EBADF); +@@ -587,38 +1386,178 @@ static struct bpf_prog *__bpf_prog_get(s + return f.file->private_data; + } + ++struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) ++{ ++ if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { ++ atomic_sub(i, &prog->aux->refcnt); ++ return ERR_PTR(-EBUSY); ++ } ++ return prog; ++} ++EXPORT_SYMBOL_GPL(bpf_prog_add); ++ ++void bpf_prog_sub(struct bpf_prog *prog, int i) ++{ ++ /* Only to be used for undoing previous bpf_prog_add() in some ++ * error path. We still know that another entity in our call ++ * path holds a reference to the program, thus atomic_sub() can ++ * be safely used in such cases! ++ */ ++ WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); ++} ++EXPORT_SYMBOL_GPL(bpf_prog_sub); ++ + struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) + { +- if (atomic_inc_return(&prog->aux->refcnt) > BPF_MAX_REFCNT) { +- atomic_dec(&prog->aux->refcnt); ++ return bpf_prog_add(prog, 1); ++} ++EXPORT_SYMBOL_GPL(bpf_prog_inc); ++ ++/* prog_idr_lock should have been held */ ++struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) ++{ ++ int refold; ++ ++ refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); ++ ++ if (refold >= BPF_MAX_REFCNT) { ++ __bpf_prog_put(prog, false); + return ERR_PTR(-EBUSY); + } ++ ++ if (!refold) ++ return ERR_PTR(-ENOENT); ++ + return prog; + } ++EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); + +-/* called by sockets/tracing/seccomp before attaching program to an event +- * pairs with bpf_prog_put() +- */ +-struct bpf_prog *bpf_prog_get(u32 ufd) ++bool bpf_prog_get_ok(struct bpf_prog *prog, ++ enum bpf_prog_type *attach_type, bool attach_drv) ++{ ++ /* not an attachment, just a refcount inc, always allow */ ++ if (!attach_type) ++ return true; ++ ++ if (prog->type != *attach_type) ++ return false; ++ if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) ++ return false; ++ ++ return true; ++} ++ ++static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, ++ bool attach_drv) + { + struct fd f = fdget(ufd); + struct bpf_prog *prog; + +- prog = __bpf_prog_get(f); ++ prog = ____bpf_prog_get(f); + if (IS_ERR(prog)) + return prog; ++ if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { ++ prog = ERR_PTR(-EINVAL); ++ goto out; ++ } + + prog = bpf_prog_inc(prog); ++out: + fdput(f); +- + return prog; + } +-EXPORT_SYMBOL_GPL(bpf_prog_get); ++ ++struct bpf_prog *bpf_prog_get(u32 ufd) ++{ ++ return __bpf_prog_get(ufd, NULL, false); ++} ++ ++struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, ++ bool attach_drv) ++{ ++ return __bpf_prog_get(ufd, &type, attach_drv); ++} ++EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); ++ ++/* Initially all BPF programs could be loaded w/o specifying ++ * expected_attach_type. Later for some of them specifying expected_attach_type ++ * at load time became required so that program could be validated properly. ++ * Programs of types that are allowed to be loaded both w/ and w/o (for ++ * backward compatibility) expected_attach_type, should have the default attach ++ * type assigned to expected_attach_type for the latter case, so that it can be ++ * validated later at attach time. ++ * ++ * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if ++ * prog type requires it but has some attach types that have to be backward ++ * compatible. ++ */ ++static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) ++{ ++ switch (attr->prog_type) { ++ case BPF_PROG_TYPE_CGROUP_SOCK: ++ /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't ++ * exist so checking for non-zero is the way to go here. ++ */ ++ if (!attr->expected_attach_type) ++ attr->expected_attach_type = ++ BPF_CGROUP_INET_SOCK_CREATE; ++ break; ++ } ++} ++ ++static int ++bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, ++ enum bpf_attach_type expected_attach_type) ++{ ++ switch (prog_type) { ++ case BPF_PROG_TYPE_CGROUP_SOCK: ++ switch (expected_attach_type) { ++ case BPF_CGROUP_INET_SOCK_CREATE: ++ case BPF_CGROUP_INET4_POST_BIND: ++ case BPF_CGROUP_INET6_POST_BIND: ++ return 0; ++ default: ++ return -EINVAL; ++ } ++ case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: ++ switch (expected_attach_type) { ++ case BPF_CGROUP_INET4_BIND: ++ case BPF_CGROUP_INET6_BIND: ++ case BPF_CGROUP_INET4_CONNECT: ++ case BPF_CGROUP_INET6_CONNECT: ++ case BPF_CGROUP_UDP4_SENDMSG: ++ case BPF_CGROUP_UDP6_SENDMSG: ++ case BPF_CGROUP_UDP4_RECVMSG: ++ case BPF_CGROUP_UDP6_RECVMSG: ++ return 0; ++ default: ++ return -EINVAL; ++ } ++ case BPF_PROG_TYPE_CGROUP_SKB: ++ switch (expected_attach_type) { ++ case BPF_CGROUP_INET_INGRESS: ++ case BPF_CGROUP_INET_EGRESS: ++ return 0; ++ default: ++ return -EINVAL; ++ } ++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: ++ switch (expected_attach_type) { ++ case BPF_CGROUP_SETSOCKOPT: ++ case BPF_CGROUP_GETSOCKOPT: ++ return 0; ++ default: ++ return -EINVAL; ++ } ++ default: ++ return 0; ++ } ++} + + /* last field in 'union bpf_attr' used by this command */ +-#define BPF_PROG_LOAD_LAST_FIELD kern_version ++#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt + +-static int bpf_prog_load(union bpf_attr *attr) ++static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) + { + enum bpf_prog_type type = attr->prog_type; + struct bpf_prog *prog; +@@ -629,8 +1568,19 @@ static int bpf_prog_load(union bpf_attr + if (CHECK_ATTR(BPF_PROG_LOAD)) + return -EINVAL; + ++ if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | ++ BPF_F_ANY_ALIGNMENT | ++ BPF_F_TEST_STATE_FREQ | ++ BPF_F_TEST_RND_HI32)) ++ return -EINVAL; ++ ++ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && ++ (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && ++ !capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ + /* copy eBPF program license from user space */ +- if (strncpy_from_user(license, u64_to_ptr(attr->license), ++ if (strncpy_from_user(license, u64_to_user_ptr(attr->license), + sizeof(license) - 1) < 0) + return -EFAULT; + license[sizeof(license) - 1] = 0; +@@ -638,30 +1588,36 @@ static int bpf_prog_load(union bpf_attr + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + is_gpl = license_is_gpl_compatible(license); + +- if (attr->insn_cnt >= BPF_MAXINSNS) +- return -EINVAL; ++ if (attr->insn_cnt == 0 || ++ attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) ++ return -E2BIG; ++ if (type != BPF_PROG_TYPE_SOCKET_FILTER && ++ type != BPF_PROG_TYPE_CGROUP_SKB && ++ !capable(CAP_SYS_ADMIN)) ++ return -EPERM; + +- if (type == BPF_PROG_TYPE_KPROBE && +- attr->kern_version != LINUX_VERSION_CODE) ++ bpf_prog_load_fixup_attach_type(attr); ++ if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) + return -EINVAL; + +- if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN)) +- return -EPERM; +- + /* plain bpf_prog allocation */ + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); + if (!prog) + return -ENOMEM; + ++ prog->expected_attach_type = attr->expected_attach_type; ++ ++ prog->aux->offload_requested = !!attr->prog_ifindex; ++ + err = bpf_prog_charge_memlock(prog); + if (err) +- goto free_prog_nouncharge; ++ goto free_prog_sec; + + prog->len = attr->insn_cnt; + + err = -EFAULT; +- if (copy_from_user(prog->insns, u64_to_ptr(attr->insns), +- prog->len * sizeof(struct bpf_insn)) != 0) ++ if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), ++ bpf_prog_insn_size(prog)) != 0) + goto free_prog; + + prog->orig_prog = NULL; +@@ -675,91 +1631,720 @@ static int bpf_prog_load(union bpf_attr + if (err < 0) + goto free_prog; + ++ prog->aux->load_time = ktime_get_boot_ns(); ++ err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); ++ if (err) ++ goto free_prog; ++ + /* run eBPF verifier */ +- err = bpf_check(&prog, attr); ++ err = bpf_check(&prog, attr, uattr); + if (err < 0) + goto free_used_maps; + +- /* fixup BPF_CALL->imm field */ +- fixup_bpf_calls(prog); +- +- /* eBPF program is ready to be JITed */ +- err = bpf_prog_select_runtime(prog); ++ prog = bpf_prog_select_runtime(prog, &err); + if (err < 0) + goto free_used_maps; + +- err = bpf_prog_new_fd(prog); +- if (err < 0) +- /* failed to allocate fd */ ++ err = bpf_prog_alloc_id(prog); ++ if (err) + goto free_used_maps; + ++ /* Upon success of bpf_prog_alloc_id(), the BPF prog is ++ * effectively publicly exposed. However, retrieving via ++ * bpf_prog_get_fd_by_id() will take another reference, ++ * therefore it cannot be gone underneath us. ++ * ++ * Only for the time /after/ successful bpf_prog_new_fd() ++ * and before returning to userspace, we might just hold ++ * one reference and any parallel close on that fd could ++ * rip everything out. Hence, below notifications must ++ * happen before bpf_prog_new_fd(). ++ * ++ * Also, any failure handling from this point onwards must ++ * be using bpf_prog_put() given the program is exposed. ++ */ ++ bpf_prog_kallsyms_add(prog); ++ ++ err = bpf_prog_new_fd(prog); ++ if (err < 0) ++ bpf_prog_put(prog); + return err; + + free_used_maps: +- free_used_maps(prog->aux); ++ /* In case we have subprogs, we need to wait for a grace ++ * period before we can tear down JIT memory since symbols ++ * are already exposed under kallsyms. ++ */ ++ __bpf_prog_put_noref(prog, prog->aux->func_cnt); ++ return err; + free_prog: + bpf_prog_uncharge_memlock(prog); +-free_prog_nouncharge: ++free_prog_sec: + bpf_prog_free(prog); + return err; + } + +-#define BPF_OBJ_LAST_FIELD bpf_fd ++#define BPF_OBJ_LAST_FIELD file_flags + + static int bpf_obj_pin(const union bpf_attr *attr) + { +- if (CHECK_ATTR(BPF_OBJ)) ++ if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) + return -EINVAL; + +- return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname)); ++ return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); + } + + static int bpf_obj_get(const union bpf_attr *attr) + { +- if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) ++ if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || ++ attr->file_flags & ~BPF_OBJ_FLAG_MASK) + return -EINVAL; + +- return bpf_obj_get_user(u64_to_ptr(attr->pathname)); ++ return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), ++ attr->file_flags); + } + +-SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) ++ ++#define BPF_PROG_ATTACH_LAST_FIELD attach_flags ++ ++#define BPF_F_ATTACH_MASK \ ++ (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) ++ ++ ++#define BPF_PROG_DETACH_LAST_FIELD attach_type ++ ++ ++#define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt ++ ++ ++#define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out ++ ++static int bpf_prog_test_run(const union bpf_attr *attr, ++ union bpf_attr __user *uattr) + { +- union bpf_attr attr = {}; +- int err; ++ struct bpf_prog *prog; ++ int ret = -ENOTSUPP; + +- if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) ++ if (!capable(CAP_SYS_ADMIN)) + return -EPERM; ++ if (CHECK_ATTR(BPF_PROG_TEST_RUN)) ++ return -EINVAL; ++ ++ if ((attr->test.ctx_size_in && !attr->test.ctx_in) || ++ (!attr->test.ctx_size_in && attr->test.ctx_in)) ++ return -EINVAL; ++ ++ if ((attr->test.ctx_size_out && !attr->test.ctx_out) || ++ (!attr->test.ctx_size_out && attr->test.ctx_out)) ++ return -EINVAL; ++ ++ prog = bpf_prog_get(attr->test.prog_fd); ++ if (IS_ERR(prog)) ++ return PTR_ERR(prog); ++ ++ if (prog->aux->ops->test_run) ++ ret = prog->aux->ops->test_run(prog, attr, uattr); ++ ++ bpf_prog_put(prog); ++ return ret; ++} ++ ++#define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id ++ ++static int bpf_obj_get_next_id(const union bpf_attr *attr, ++ union bpf_attr __user *uattr, ++ struct idr *idr, ++ spinlock_t *lock) ++{ ++ u32 next_id = attr->start_id; ++ int err = 0; ++ ++ if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) ++ return -EINVAL; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ next_id++; ++ spin_lock_bh(lock); ++ if (!idr_get_next(idr, &next_id)) ++ err = -ENOENT; ++ spin_unlock_bh(lock); ++ ++ if (!err) ++ err = put_user(next_id, &uattr->next_id); ++ ++ return err; ++} ++ ++#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id ++ ++static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) ++{ ++ struct bpf_prog *prog; ++ u32 id = attr->prog_id; ++ int fd; ++ ++ if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) ++ return -EINVAL; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ spin_lock_bh(&prog_idr_lock); ++ prog = idr_find(&prog_idr, id); ++ if (prog) ++ prog = bpf_prog_inc_not_zero(prog); ++ else ++ prog = ERR_PTR(-ENOENT); ++ spin_unlock_bh(&prog_idr_lock); ++ ++ if (IS_ERR(prog)) ++ return PTR_ERR(prog); ++ ++ fd = bpf_prog_new_fd(prog); ++ if (fd < 0) ++ bpf_prog_put(prog); ++ ++ return fd; ++} ++ ++#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags ++ ++static int bpf_map_get_fd_by_id(const union bpf_attr *attr) ++{ ++ struct bpf_map *map; ++ u32 id = attr->map_id; ++ int f_flags; ++ int fd; ++ ++ if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || ++ attr->open_flags & ~BPF_OBJ_FLAG_MASK) ++ return -EINVAL; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ f_flags = bpf_get_file_flag(attr->open_flags); ++ if (f_flags < 0) ++ return f_flags; ++ ++ spin_lock_bh(&map_idr_lock); ++ map = idr_find(&map_idr, id); ++ if (map) ++ map = __bpf_map_inc_not_zero(map, true); ++ else ++ map = ERR_PTR(-ENOENT); ++ spin_unlock_bh(&map_idr_lock); ++ ++ if (IS_ERR(map)) ++ return PTR_ERR(map); + +- if (!access_ok(VERIFY_READ, uattr, 1)) ++ fd = bpf_map_new_fd(map, f_flags); ++ if (fd < 0) ++ bpf_map_put_with_uref(map); ++ ++ return fd; ++} ++ ++static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, ++ unsigned long addr, u32 *off, ++ u32 *type) ++{ ++ const struct bpf_map *map; ++ int i; ++ ++ for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { ++ map = prog->aux->used_maps[i]; ++ if (map == (void *)addr) { ++ *type = BPF_PSEUDO_MAP_FD; ++ return map; ++ } ++ if (!map->ops->map_direct_value_meta) ++ continue; ++ if (!map->ops->map_direct_value_meta(map, addr, off)) { ++ *type = BPF_PSEUDO_MAP_VALUE; ++ return map; ++ } ++ } ++ ++ return NULL; ++} ++ ++static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, ++ const struct cred *f_cred) ++{ ++ const struct bpf_map *map; ++ struct bpf_insn *insns; ++ u32 off, type; ++ u64 imm; ++ int i; ++ ++ insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), ++ GFP_USER); ++ if (!insns) ++ return insns; ++ ++ for (i = 0; i < prog->len; i++) { ++ if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { ++ insns[i].code = BPF_JMP | BPF_CALL; ++ insns[i].imm = BPF_FUNC_tail_call; ++ /* fall-through */ ++ } ++ if (insns[i].code == (BPF_JMP | BPF_CALL) || ++ insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { ++ if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) ++ insns[i].code = BPF_JMP | BPF_CALL; ++ if (!bpf_dump_raw_ok(f_cred)) ++ insns[i].imm = 0; ++ continue; ++ } ++ ++ if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) ++ continue; ++ ++ imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; ++ map = bpf_map_from_imm(prog, imm, &off, &type); ++ if (map) { ++ insns[i].src_reg = type; ++ insns[i].imm = map->id; ++ insns[i + 1].imm = off; ++ continue; ++ } ++ } ++ ++ return insns; ++} ++ ++static int set_info_rec_size(struct bpf_prog_info *info) ++{ ++ /* ++ * Ensure info.*_rec_size is the same as kernel expected size ++ * ++ * or ++ * ++ * Only allow zero *_rec_size if both _rec_size and _cnt are ++ * zero. In this case, the kernel will set the expected ++ * _rec_size back to the info. ++ */ ++ ++ if ((info->nr_func_info || info->func_info_rec_size) && ++ info->func_info_rec_size != sizeof(struct bpf_func_info)) ++ return -EINVAL; ++ ++ if ((info->nr_line_info || info->line_info_rec_size) && ++ info->line_info_rec_size != sizeof(struct bpf_line_info)) ++ return -EINVAL; ++ ++ if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && ++ info->jited_line_info_rec_size != sizeof(__u64)) ++ return -EINVAL; ++ ++ info->func_info_rec_size = sizeof(struct bpf_func_info); ++ info->line_info_rec_size = sizeof(struct bpf_line_info); ++ info->jited_line_info_rec_size = sizeof(__u64); ++ ++ return 0; ++} ++ ++static int bpf_prog_get_info_by_fd(struct file *file, ++ struct bpf_prog *prog, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); ++ struct bpf_prog_info info; ++ u32 info_len = attr->info.info_len; ++ struct bpf_prog_stats stats; ++ char __user *uinsns; ++ u32 ulen; ++ int err; ++ ++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); ++ if (err) ++ return err; ++ info_len = min_t(u32, sizeof(info), info_len); ++ ++ memset(&info, 0, sizeof(info)); ++ if (copy_from_user(&info, uinfo, info_len)) + return -EFAULT; + +- if (size > PAGE_SIZE) /* silly large */ +- return -E2BIG; ++ info.type = prog->type; ++ info.id = prog->aux->id; ++ info.load_time = prog->aux->load_time; ++ info.created_by_uid = from_kuid_munged(current_user_ns(), ++ prog->aux->user->uid); ++ info.gpl_compatible = prog->gpl_compatible; ++ ++ memcpy(info.tag, prog->tag, sizeof(prog->tag)); ++ memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); ++ ++ ulen = info.nr_map_ids; ++ info.nr_map_ids = prog->aux->used_map_cnt; ++ ulen = min_t(u32, info.nr_map_ids, ulen); ++ if (ulen) { ++ u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); ++ u32 i; ++ ++ for (i = 0; i < ulen; i++) ++ if (put_user(prog->aux->used_maps[i]->id, ++ &user_map_ids[i])) ++ return -EFAULT; ++ } ++ ++ err = set_info_rec_size(&info); ++ if (err) ++ return err; ++ ++ bpf_prog_get_stats(prog, &stats); ++ info.run_time_ns = stats.nsecs; ++ info.run_cnt = stats.cnt; ++ ++ if (!capable(CAP_SYS_ADMIN)) { ++ info.jited_prog_len = 0; ++ info.xlated_prog_len = 0; ++ info.nr_jited_ksyms = 0; ++ info.nr_jited_func_lens = 0; ++ info.nr_func_info = 0; ++ info.nr_line_info = 0; ++ info.nr_jited_line_info = 0; ++ goto done; ++ } ++ ++ ulen = info.xlated_prog_len; ++ info.xlated_prog_len = bpf_prog_insn_size(prog); ++ if (info.xlated_prog_len && ulen) { ++ struct bpf_insn *insns_sanitized; ++ bool fault; ++ ++ if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { ++ info.xlated_prog_insns = 0; ++ goto done; ++ } ++ insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); ++ if (!insns_sanitized) ++ return -ENOMEM; ++ uinsns = u64_to_user_ptr(info.xlated_prog_insns); ++ ulen = min_t(u32, info.xlated_prog_len, ulen); ++ fault = copy_to_user(uinsns, insns_sanitized, ulen); ++ kfree(insns_sanitized); ++ if (fault) ++ return -EFAULT; ++ } ++ ++ /* NOTE: the following code is supposed to be skipped for offload. ++ * bpf_prog_offload_info_fill() is the place to fill similar fields ++ * for offload. ++ */ ++ ulen = info.jited_prog_len; ++ if (prog->aux->func_cnt) { ++ u32 i; ++ ++ info.jited_prog_len = 0; ++ for (i = 0; i < prog->aux->func_cnt; i++) ++ info.jited_prog_len += prog->aux->func[i]->jited_len; ++ } else { ++ info.jited_prog_len = prog->jited_len; ++ } ++ ++ if (info.jited_prog_len && ulen) { ++ if (bpf_dump_raw_ok(file->f_cred)) { ++ uinsns = u64_to_user_ptr(info.jited_prog_insns); ++ ulen = min_t(u32, info.jited_prog_len, ulen); ++ ++ /* for multi-function programs, copy the JITed ++ * instructions for all the functions ++ */ ++ if (prog->aux->func_cnt) { ++ u32 len, free, i; ++ u8 *img; ++ ++ free = ulen; ++ for (i = 0; i < prog->aux->func_cnt; i++) { ++ len = prog->aux->func[i]->jited_len; ++ len = min_t(u32, len, free); ++ img = (u8 *) prog->aux->func[i]->bpf_func; ++ if (copy_to_user(uinsns, img, len)) ++ return -EFAULT; ++ uinsns += len; ++ free -= len; ++ if (!free) ++ break; ++ } ++ } else { ++ if (copy_to_user(uinsns, prog->bpf_func, ulen)) ++ return -EFAULT; ++ } ++ } else { ++ info.jited_prog_insns = 0; ++ } ++ } ++ ++ ulen = info.nr_jited_ksyms; ++ info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; ++ if (ulen) { ++ if (bpf_dump_raw_ok(file->f_cred)) { ++ unsigned long ksym_addr; ++ u64 __user *user_ksyms; ++ u32 i; ++ ++ /* copy the address of the kernel symbol ++ * corresponding to each function ++ */ ++ ulen = min_t(u32, info.nr_jited_ksyms, ulen); ++ user_ksyms = u64_to_user_ptr(info.jited_ksyms); ++ if (prog->aux->func_cnt) { ++ for (i = 0; i < ulen; i++) { ++ ksym_addr = (unsigned long) ++ prog->aux->func[i]->bpf_func; ++ if (put_user((u64) ksym_addr, ++ &user_ksyms[i])) ++ return -EFAULT; ++ } ++ } else { ++ ksym_addr = (unsigned long) prog->bpf_func; ++ if (put_user((u64) ksym_addr, &user_ksyms[0])) ++ return -EFAULT; ++ } ++ } else { ++ info.jited_ksyms = 0; ++ } ++ } ++ ++ ulen = info.nr_jited_func_lens; ++ info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; ++ if (ulen) { ++ if (bpf_dump_raw_ok(file->f_cred)) { ++ u32 __user *user_lens; ++ u32 func_len, i; ++ ++ /* copy the JITed image lengths for each function */ ++ ulen = min_t(u32, info.nr_jited_func_lens, ulen); ++ user_lens = u64_to_user_ptr(info.jited_func_lens); ++ if (prog->aux->func_cnt) { ++ for (i = 0; i < ulen; i++) { ++ func_len = ++ prog->aux->func[i]->jited_len; ++ if (put_user(func_len, &user_lens[i])) ++ return -EFAULT; ++ } ++ } else { ++ func_len = prog->jited_len; ++ if (put_user(func_len, &user_lens[0])) ++ return -EFAULT; ++ } ++ } else { ++ info.jited_func_lens = 0; ++ } ++ } ++ ++ if (prog->aux->btf) ++ info.btf_id = btf_id(prog->aux->btf); ++ ++ ulen = info.nr_func_info; ++ info.nr_func_info = prog->aux->func_info_cnt; ++ if (info.nr_func_info && ulen) { ++ char __user *user_finfo; ++ ++ user_finfo = u64_to_user_ptr(info.func_info); ++ ulen = min_t(u32, info.nr_func_info, ulen); ++ if (copy_to_user(user_finfo, prog->aux->func_info, ++ info.func_info_rec_size * ulen)) ++ return -EFAULT; ++ } ++ ++ ulen = info.nr_line_info; ++ info.nr_line_info = prog->aux->nr_linfo; ++ if (info.nr_line_info && ulen) { ++ __u8 __user *user_linfo; ++ ++ user_linfo = u64_to_user_ptr(info.line_info); ++ ulen = min_t(u32, info.nr_line_info, ulen); ++ if (copy_to_user(user_linfo, prog->aux->linfo, ++ info.line_info_rec_size * ulen)) ++ return -EFAULT; ++ } + +- /* If we're handed a bigger struct than we know of, +- * ensure all the unknown bits are 0 - i.e. new +- * user-space does not rely on any kernel feature +- * extensions we dont know about yet. +- */ +- if (size > sizeof(attr)) { +- unsigned char __user *addr; +- unsigned char __user *end; +- unsigned char val; +- +- addr = (void __user *)uattr + sizeof(attr); +- end = (void __user *)uattr + size; +- +- for (; addr < end; addr++) { +- err = get_user(val, addr); +- if (err) +- return err; +- if (val) +- return -E2BIG; ++ ulen = info.nr_jited_line_info; ++ if (prog->aux->jited_linfo) ++ info.nr_jited_line_info = prog->aux->nr_linfo; ++ else ++ info.nr_jited_line_info = 0; ++ if (info.nr_jited_line_info && ulen) { ++ if (bpf_dump_raw_ok(file->f_cred)) { ++ __u64 __user *user_linfo; ++ u32 i; ++ ++ user_linfo = u64_to_user_ptr(info.jited_line_info); ++ ulen = min_t(u32, info.nr_jited_line_info, ulen); ++ for (i = 0; i < ulen; i++) { ++ if (put_user((__u64)(long)prog->aux->jited_linfo[i], ++ &user_linfo[i])) ++ return -EFAULT; ++ } ++ } else { ++ info.jited_line_info = 0; + } +- size = sizeof(attr); + } + ++ ulen = info.nr_prog_tags; ++ info.nr_prog_tags = prog->aux->func_cnt ? : 1; ++ if (ulen) { ++ __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; ++ u32 i; ++ ++ user_prog_tags = u64_to_user_ptr(info.prog_tags); ++ ulen = min_t(u32, info.nr_prog_tags, ulen); ++ if (prog->aux->func_cnt) { ++ for (i = 0; i < ulen; i++) { ++ if (copy_to_user(user_prog_tags[i], ++ prog->aux->func[i]->tag, ++ BPF_TAG_SIZE)) ++ return -EFAULT; ++ } ++ } else { ++ if (copy_to_user(user_prog_tags[0], ++ prog->tag, BPF_TAG_SIZE)) ++ return -EFAULT; ++ } ++ } ++ ++done: ++ if (copy_to_user(uinfo, &info, info_len) || ++ put_user(info_len, &uattr->info.info_len)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int bpf_map_get_info_by_fd(struct file *file, ++ struct bpf_map *map, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); ++ struct bpf_map_info info; ++ u32 info_len = attr->info.info_len; ++ int err; ++ ++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); ++ if (err) ++ return err; ++ info_len = min_t(u32, sizeof(info), info_len); ++ ++ memset(&info, 0, sizeof(info)); ++ info.type = map->map_type; ++ info.id = map->id; ++ info.key_size = map->key_size; ++ info.value_size = map->value_size; ++ info.max_entries = map->max_entries; ++ info.map_flags = map->map_flags; ++ memcpy(info.name, map->name, sizeof(map->name)); ++ ++ if (map->btf) { ++ info.btf_id = btf_id(map->btf); ++ info.btf_key_type_id = map->btf_key_type_id; ++ info.btf_value_type_id = map->btf_value_type_id; ++ } ++ ++ if (copy_to_user(uinfo, &info, info_len) || ++ put_user(info_len, &uattr->info.info_len)) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++static int bpf_btf_get_info_by_fd(struct file *file, ++ struct btf *btf, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); ++ u32 info_len = attr->info.info_len; ++ int err; ++ ++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); ++ if (err) ++ return err; ++ ++ return btf_get_info_by_fd(btf, attr, uattr); ++} ++ ++#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info ++ ++static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ int ufd = attr->info.bpf_fd; ++ struct fd f; ++ int err; ++ ++ if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) ++ return -EINVAL; ++ ++ f = fdget(ufd); ++ if (!f.file) ++ return -EBADFD; ++ ++ if (f.file->f_op == &bpf_prog_fops) ++ err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, ++ uattr); ++ else if (f.file->f_op == &bpf_map_fops) ++ err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, ++ uattr); ++ else if (f.file->f_op == &btf_fops) ++ err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); ++ else ++ err = -EINVAL; ++ ++ fdput(f); ++ return err; ++} ++ ++#define BPF_BTF_LOAD_LAST_FIELD btf_log_level ++ ++static int bpf_btf_load(const union bpf_attr *attr) ++{ ++ if (CHECK_ATTR(BPF_BTF_LOAD)) ++ return -EINVAL; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ return btf_new_fd(attr); ++} ++ ++#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id ++ ++static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) ++{ ++ if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) ++ return -EINVAL; ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ return btf_get_fd_by_id(attr->btf_id); ++} ++ ++ ++#define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr ++ ++SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) ++{ ++ union bpf_attr attr; ++ int err; ++ ++ if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); ++ if (err) ++ return err; ++ size = min_t(u32, size, sizeof(attr)); ++ + /* copy attributes from user space, may be less than sizeof(bpf_attr) */ ++ memset(&attr, 0, sizeof(attr)); + if (copy_from_user(&attr, uattr, size) != 0) + return -EFAULT; + +@@ -779,8 +2364,11 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf + case BPF_MAP_GET_NEXT_KEY: + err = map_get_next_key(&attr); + break; ++ case BPF_MAP_FREEZE: ++ err = map_freeze(&attr); ++ break; + case BPF_PROG_LOAD: +- err = bpf_prog_load(&attr); ++ err = bpf_prog_load(&attr, uattr); + break; + case BPF_OBJ_PIN: + err = bpf_obj_pin(&attr); +@@ -788,6 +2376,39 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf + case BPF_OBJ_GET: + err = bpf_obj_get(&attr); + break; ++ case BPF_PROG_TEST_RUN: ++ err = bpf_prog_test_run(&attr, uattr); ++ break; ++ case BPF_PROG_GET_NEXT_ID: ++ err = bpf_obj_get_next_id(&attr, uattr, ++ &prog_idr, &prog_idr_lock); ++ break; ++ case BPF_MAP_GET_NEXT_ID: ++ err = bpf_obj_get_next_id(&attr, uattr, ++ &map_idr, &map_idr_lock); ++ break; ++ case BPF_BTF_GET_NEXT_ID: ++ err = bpf_obj_get_next_id(&attr, uattr, ++ &btf_idr, &btf_idr_lock); ++ break; ++ case BPF_PROG_GET_FD_BY_ID: ++ err = bpf_prog_get_fd_by_id(&attr); ++ break; ++ case BPF_MAP_GET_FD_BY_ID: ++ err = bpf_map_get_fd_by_id(&attr); ++ break; ++ case BPF_OBJ_GET_INFO_BY_FD: ++ err = bpf_obj_get_info_by_fd(&attr, uattr); ++ break; ++ case BPF_BTF_LOAD: ++ err = bpf_btf_load(&attr); ++ break; ++ case BPF_BTF_GET_FD_BY_ID: ++ err = bpf_btf_get_fd_by_id(&attr); ++ break; ++ case BPF_MAP_LOOKUP_AND_DELETE_ELEM: ++ err = map_lookup_and_delete_elem(&attr); ++ break; + default: + err = -EINVAL; + break; +--- /dev/null ++++ b/kernel/bpf/sysfs_btf.c +@@ -0,0 +1,45 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Provide kernel BTF information for introspection and use by eBPF tools. ++ */ ++#include ++#include ++#include ++#include ++#include ++ ++/* See scripts/link-vmlinux.sh, gen_btf() func for details */ ++extern char __weak __start_BTF[]; ++extern char __weak __stop_BTF[]; ++ ++static ssize_t ++btf_vmlinux_read(struct file *file, struct kobject *kobj, ++ struct bin_attribute *bin_attr, ++ char *buf, loff_t off, size_t len) ++{ ++ memcpy(buf, __start_BTF + off, len); ++ return len; ++} ++ ++static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { ++ .attr = { .name = "vmlinux", .mode = 0444, }, ++ .read = btf_vmlinux_read, ++}; ++ ++static struct kobject *btf_kobj; ++ ++static int __init btf_vmlinux_init(void) ++{ ++ bin_attr_btf_vmlinux.size = __stop_BTF - __start_BTF; ++ ++ if (!__start_BTF || bin_attr_btf_vmlinux.size == 0) ++ return 0; ++ ++ btf_kobj = kobject_create_and_add("btf", kernel_kobj); ++ if (!btf_kobj) ++ return -ENOMEM; ++ ++ return sysfs_create_bin_file(btf_kobj, &bin_attr_btf_vmlinux); ++} ++ ++subsys_initcall(btf_vmlinux_init); +--- /dev/null ++++ b/kernel/bpf/tnum.c +@@ -0,0 +1,196 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* tnum: tracked (or tristate) numbers ++ * ++ * A tnum tracks knowledge about the bits of a value. Each bit can be either ++ * known (0 or 1), or unknown (x). Arithmetic operations on tnums will ++ * propagate the unknown bits such that the tnum result represents all the ++ * possible results for possible values of the operands. ++ */ ++#include ++#include ++ ++#define TNUM(_v, _m) (struct tnum){.value = _v, .mask = _m} ++/* A completely unknown value */ ++const struct tnum tnum_unknown = { .value = 0, .mask = -1 }; ++ ++struct tnum tnum_const(u64 value) ++{ ++ return TNUM(value, 0); ++} ++ ++struct tnum tnum_range(u64 min, u64 max) ++{ ++ u64 chi = min ^ max, delta; ++ u8 bits = fls64(chi); ++ ++ /* special case, needed because 1ULL << 64 is undefined */ ++ if (bits > 63) ++ return tnum_unknown; ++ /* e.g. if chi = 4, bits = 3, delta = (1<<3) - 1 = 7. ++ * if chi = 0, bits = 0, delta = (1<<0) - 1 = 0, so we return ++ * constant min (since min == max). ++ */ ++ delta = (1ULL << bits) - 1; ++ return TNUM(min & ~delta, delta); ++} ++ ++struct tnum tnum_lshift(struct tnum a, u8 shift) ++{ ++ return TNUM(a.value << shift, a.mask << shift); ++} ++ ++struct tnum tnum_rshift(struct tnum a, u8 shift) ++{ ++ return TNUM(a.value >> shift, a.mask >> shift); ++} ++ ++struct tnum tnum_arshift(struct tnum a, u8 min_shift, u8 insn_bitness) ++{ ++ /* if a.value is negative, arithmetic shifting by minimum shift ++ * will have larger negative offset compared to more shifting. ++ * If a.value is nonnegative, arithmetic shifting by minimum shift ++ * will have larger positive offset compare to more shifting. ++ */ ++ if (insn_bitness == 32) ++ return TNUM((u32)(((s32)a.value) >> min_shift), ++ (u32)(((s32)a.mask) >> min_shift)); ++ else ++ return TNUM((s64)a.value >> min_shift, ++ (s64)a.mask >> min_shift); ++} ++ ++struct tnum tnum_add(struct tnum a, struct tnum b) ++{ ++ u64 sm, sv, sigma, chi, mu; ++ ++ sm = a.mask + b.mask; ++ sv = a.value + b.value; ++ sigma = sm + sv; ++ chi = sigma ^ sv; ++ mu = chi | a.mask | b.mask; ++ return TNUM(sv & ~mu, mu); ++} ++ ++struct tnum tnum_sub(struct tnum a, struct tnum b) ++{ ++ u64 dv, alpha, beta, chi, mu; ++ ++ dv = a.value - b.value; ++ alpha = dv + a.mask; ++ beta = dv - b.mask; ++ chi = alpha ^ beta; ++ mu = chi | a.mask | b.mask; ++ return TNUM(dv & ~mu, mu); ++} ++ ++struct tnum tnum_and(struct tnum a, struct tnum b) ++{ ++ u64 alpha, beta, v; ++ ++ alpha = a.value | a.mask; ++ beta = b.value | b.mask; ++ v = a.value & b.value; ++ return TNUM(v, alpha & beta & ~v); ++} ++ ++struct tnum tnum_or(struct tnum a, struct tnum b) ++{ ++ u64 v, mu; ++ ++ v = a.value | b.value; ++ mu = a.mask | b.mask; ++ return TNUM(v, mu & ~v); ++} ++ ++struct tnum tnum_xor(struct tnum a, struct tnum b) ++{ ++ u64 v, mu; ++ ++ v = a.value ^ b.value; ++ mu = a.mask | b.mask; ++ return TNUM(v & ~mu, mu); ++} ++ ++/* half-multiply add: acc += (unknown * mask * value). ++ * An intermediate step in the multiply algorithm. ++ */ ++static struct tnum hma(struct tnum acc, u64 value, u64 mask) ++{ ++ while (mask) { ++ if (mask & 1) ++ acc = tnum_add(acc, TNUM(0, value)); ++ mask >>= 1; ++ value <<= 1; ++ } ++ return acc; ++} ++ ++struct tnum tnum_mul(struct tnum a, struct tnum b) ++{ ++ struct tnum acc; ++ u64 pi; ++ ++ pi = a.value * b.value; ++ acc = hma(TNUM(pi, 0), a.mask, b.mask | b.value); ++ return hma(acc, b.mask, a.value); ++} ++ ++/* Note that if a and b disagree - i.e. one has a 'known 1' where the other has ++ * a 'known 0' - this will return a 'known 1' for that bit. ++ */ ++struct tnum tnum_intersect(struct tnum a, struct tnum b) ++{ ++ u64 v, mu; ++ ++ v = a.value | b.value; ++ mu = a.mask & b.mask; ++ return TNUM(v & ~mu, mu); ++} ++ ++struct tnum tnum_cast(struct tnum a, u8 size) ++{ ++ a.value &= (1ULL << (size * 8)) - 1; ++ a.mask &= (1ULL << (size * 8)) - 1; ++ return a; ++} ++ ++bool tnum_is_aligned(struct tnum a, u64 size) ++{ ++ if (!size) ++ return true; ++ return !((a.value | a.mask) & (size - 1)); ++} ++ ++bool tnum_in(struct tnum a, struct tnum b) ++{ ++ if (b.mask & ~a.mask) ++ return false; ++ b.value &= ~a.mask; ++ return a.value == b.value; ++} ++ ++int tnum_strn(char *str, size_t size, struct tnum a) ++{ ++ return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask); ++} ++EXPORT_SYMBOL_GPL(tnum_strn); ++ ++int tnum_sbin(char *str, size_t size, struct tnum a) ++{ ++ size_t n; ++ ++ for (n = 64; n; n--) { ++ if (n < size) { ++ if (a.mask & 1) ++ str[n - 1] = 'x'; ++ else if (a.value & 1) ++ str[n - 1] = '1'; ++ else ++ str[n - 1] = '0'; ++ } ++ a.mask >>= 1; ++ a.value >>= 1; ++ } ++ str[min(size - 1, (size_t)64)] = 0; ++ return 64; ++} +--- a/kernel/bpf/verifier.c ++++ b/kernel/bpf/verifier.c +@@ -1,22 +1,36 @@ ++// SPDX-License-Identifier: GPL-2.0-only + /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com +- * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of version 2 of the GNU General Public +- * License as published by the Free Software Foundation. +- * +- * This program is distributed in the hope that it will be useful, but +- * WITHOUT ANY WARRANTY; without even the implied warranty of +- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +- * General Public License for more details. ++ * Copyright (c) 2016 Facebook ++ * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io + */ ++#include + #include + #include + #include + #include ++#include ++#include + #include + #include + #include + #include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "disasm.h" ++ ++static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { ++#define BPF_PROG_TYPE(_id, _name) \ ++ [_id] = & _name ## _verifier_ops, ++#define BPF_MAP_TYPE(_id, _ops) ++#include ++#undef BPF_PROG_TYPE ++#undef BPF_MAP_TYPE ++}; + + /* bpf_check() is a static code analyzer that walks eBPF program + * instruction by instruction and updates register/stack state. +@@ -30,7 +44,7 @@ + * - out of bounds or malformed jumps + * The second pass is all possible path descent from the 1st insn. + * Since it's analyzing all pathes through the program, the length of the +- * analysis is limited to 32k insn, which may be hit even if total number of ++ * analysis is limited to 64k insn, which may be hit even if total number of + * insn is less then 4K, but there are too many branches that change stack/regs. + * Number of 'branches to be analyzed' is limited to 1k + * +@@ -58,13 +72,13 @@ + * (and -20 constant is saved for further stack bounds checking). + * Meaning that this reg is a pointer to stack plus known immediate constant. + * +- * Most of the time the registers have UNKNOWN_VALUE type, which ++ * Most of the time the registers have SCALAR_VALUE type, which + * means the register has some value, but it's not a valid pointer. +- * (like pointer plus pointer becomes UNKNOWN_VALUE type) ++ * (like pointer plus pointer becomes SCALAR_VALUE type) + * + * When verifier sees load or store instructions the type of base register +- * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, FRAME_PTR. These are three pointer +- * types recognized by check_mem_access() function. ++ * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are ++ * four pointer types recognized by check_mem_access() function. + * + * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' + * and the range of [ptr, ptr + map's value_size) is accessible. +@@ -123,346 +137,713 @@ + * + * After the call R0 is set to return type of the function and registers R1-R5 + * are set to NOT_INIT to indicate that they are no longer readable. ++ * ++ * The following reference types represent a potential reference to a kernel ++ * resource which, after first being allocated, must be checked and freed by ++ * the BPF program: ++ * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET ++ * ++ * When the verifier sees a helper call return a reference type, it allocates a ++ * pointer id for the reference and stores it in the current function state. ++ * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into ++ * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type ++ * passes through a NULL-check conditional. For the branch wherein the state is ++ * changed to CONST_IMM, the verifier releases the reference. ++ * ++ * For each helper function that allocates a reference, such as ++ * bpf_sk_lookup_tcp(), there is a corresponding release function, such as ++ * bpf_sk_release(). When a reference type passes into the release function, ++ * the verifier also releases the reference. If any unchecked or unreleased ++ * reference remains at the end of the program, the verifier rejects it. + */ + +-/* types of values stored in eBPF registers */ +-enum bpf_reg_type { +- NOT_INIT = 0, /* nothing was written into register */ +- UNKNOWN_VALUE, /* reg doesn't contain a valid pointer */ +- PTR_TO_CTX, /* reg points to bpf_context */ +- CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ +- PTR_TO_MAP_VALUE, /* reg points to map element value */ +- PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ +- FRAME_PTR, /* reg == frame_pointer */ +- PTR_TO_STACK, /* reg == frame_pointer + imm */ +- CONST_IMM, /* constant integer value */ +-}; +- +-struct reg_state { +- enum bpf_reg_type type; +- union { +- /* valid when type == CONST_IMM | PTR_TO_STACK */ +- int imm; +- +- /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | +- * PTR_TO_MAP_VALUE_OR_NULL +- */ +- struct bpf_map *map_ptr; +- }; +-}; +- +-enum bpf_stack_slot_type { +- STACK_INVALID, /* nothing was stored in this stack slot */ +- STACK_SPILL, /* register spilled into stack */ +- STACK_MISC /* BPF program wrote some data into this slot */ +-}; +- +-#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +- +-/* state of the program: +- * type of all registers and stack info +- */ +-struct verifier_state { +- struct reg_state regs[MAX_BPF_REG]; +- u8 stack_slot_type[MAX_BPF_STACK]; +- struct reg_state spilled_regs[MAX_BPF_STACK / BPF_REG_SIZE]; +-}; +- +-/* linked list of verifier states used to prune search */ +-struct verifier_state_list { +- struct verifier_state state; +- struct verifier_state_list *next; +-}; +- + /* verifier_state + insn_idx are pushed to stack when branch is encountered */ +-struct verifier_stack_elem { ++struct bpf_verifier_stack_elem { + /* verifer state is 'st' + * before processing instruction 'insn_idx' + * and after processing instruction 'prev_insn_idx' + */ +- struct verifier_state st; ++ struct bpf_verifier_state st; + int insn_idx; + int prev_insn_idx; +- struct verifier_stack_elem *next; ++ struct bpf_verifier_stack_elem *next; + }; + +-#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ ++#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 ++#define BPF_COMPLEXITY_LIMIT_STATES 64 + +-/* single container for all structs +- * one verifier_env per bpf_check() call +- */ +-struct verifier_env { +- struct bpf_prog *prog; /* eBPF program being verified */ +- struct verifier_stack_elem *head; /* stack of verifier states to be processed */ +- int stack_size; /* number of states to be processed */ +- struct verifier_state cur_state; /* current verifier state */ +- struct verifier_state_list **explored_states; /* search pruning optimization */ +- struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */ +- u32 used_map_cnt; /* number of used maps */ +- bool allow_ptr_leaks; +-}; ++#define BPF_MAP_PTR_UNPRIV 1UL ++#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ ++ POISON_POINTER_DELTA)) ++#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) + +-/* verbose verifier prints what it's seeing +- * bpf_check() is called under lock, so no race to access these global vars +- */ +-static u32 log_level, log_size, log_len; +-static char *log_buf; ++static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) ++{ ++ return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON; ++} ++ ++static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) ++{ ++ return aux->map_state & BPF_MAP_PTR_UNPRIV; ++} ++ ++static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, ++ const struct bpf_map *map, bool unpriv) ++{ ++ BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); ++ unpriv |= bpf_map_ptr_unpriv(aux); ++ aux->map_state = (unsigned long)map | ++ (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); ++} ++ ++struct bpf_call_arg_meta { ++ struct bpf_map *map_ptr; ++ bool raw_mode; ++ bool pkt_access; ++ int regno; ++ int access_size; ++ u64 msize_max_value; ++ int ref_obj_id; ++ int func_id; ++}; + + static DEFINE_MUTEX(bpf_verifier_lock); + ++static const struct bpf_line_info * ++find_linfo(const struct bpf_verifier_env *env, u32 insn_off) ++{ ++ const struct bpf_line_info *linfo; ++ const struct bpf_prog *prog; ++ u32 i, nr_linfo; ++ ++ prog = env->prog; ++ nr_linfo = prog->aux->nr_linfo; ++ ++ if (!nr_linfo || insn_off >= prog->len) ++ return NULL; ++ ++ linfo = prog->aux->linfo; ++ for (i = 1; i < nr_linfo; i++) ++ if (insn_off < linfo[i].insn_off) ++ break; ++ ++ return &linfo[i - 1]; ++} ++ ++void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, ++ va_list args) ++{ ++ unsigned int n; ++ ++ n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); ++ ++ WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, ++ "verifier log line truncated - local buffer too short\n"); ++ ++ n = min(log->len_total - log->len_used - 1, n); ++ log->kbuf[n] = '\0'; ++ ++ if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) ++ log->len_used += n; ++ else ++ log->ubuf = NULL; ++} ++ + /* log_level controls verbosity level of eBPF verifier. +- * verbose() is used to dump the verification trace to the log, so the user +- * can figure out what's wrong with the program ++ * bpf_verifier_log_write() is used to dump the verification trace to the log, ++ * so the user can figure out what's wrong with the program + */ +-static __printf(1, 2) void verbose(const char *fmt, ...) ++__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, ++ const char *fmt, ...) + { + va_list args; + +- if (log_level == 0 || log_len >= log_size - 1) ++ if (!bpf_verifier_log_needed(&env->log)) + return; + + va_start(args, fmt); +- log_len += vscnprintf(log_buf + log_len, log_size - log_len, fmt, args); ++ bpf_verifier_vlog(&env->log, fmt, args); + va_end(args); + } ++EXPORT_SYMBOL_GPL(bpf_verifier_log_write); ++ ++__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) ++{ ++ struct bpf_verifier_env *env = private_data; ++ va_list args; ++ ++ if (!bpf_verifier_log_needed(&env->log)) ++ return; ++ ++ va_start(args, fmt); ++ bpf_verifier_vlog(&env->log, fmt, args); ++ va_end(args); ++} ++ ++static const char *ltrim(const char *s) ++{ ++ while (isspace(*s)) ++ s++; ++ ++ return s; ++} ++ ++__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, ++ u32 insn_off, ++ const char *prefix_fmt, ...) ++{ ++ const struct bpf_line_info *linfo; ++ ++ if (!bpf_verifier_log_needed(&env->log)) ++ return; ++ ++ linfo = find_linfo(env, insn_off); ++ if (!linfo || linfo == env->prev_linfo) ++ return; ++ ++ if (prefix_fmt) { ++ va_list args; ++ ++ va_start(args, prefix_fmt); ++ bpf_verifier_vlog(&env->log, prefix_fmt, args); ++ va_end(args); ++ } ++ ++ verbose(env, "%s\n", ++ ltrim(btf_name_by_offset(env->prog->aux->btf, ++ linfo->line_off))); ++ ++ env->prev_linfo = linfo; ++} ++ ++static bool type_is_pkt_pointer(enum bpf_reg_type type) ++{ ++ return type == PTR_TO_PACKET || ++ type == PTR_TO_PACKET_META; ++} ++ ++static bool type_is_sk_pointer(enum bpf_reg_type type) ++{ ++ return type == PTR_TO_SOCKET || ++ type == PTR_TO_SOCK_COMMON || ++ type == PTR_TO_TCP_SOCK || ++ type == PTR_TO_XDP_SOCK; ++} ++ ++static bool reg_type_may_be_null(enum bpf_reg_type type) ++{ ++ return type == PTR_TO_MAP_VALUE_OR_NULL || ++ type == PTR_TO_SOCKET_OR_NULL || ++ type == PTR_TO_SOCK_COMMON_OR_NULL || ++ type == PTR_TO_TCP_SOCK_OR_NULL; ++} ++ ++static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) ++{ ++ return reg->type == PTR_TO_MAP_VALUE && ++ map_value_has_spin_lock(reg->map_ptr); ++} ++ ++static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) ++{ ++ return type == PTR_TO_SOCKET || ++ type == PTR_TO_SOCKET_OR_NULL || ++ type == PTR_TO_TCP_SOCK || ++ type == PTR_TO_TCP_SOCK_OR_NULL; ++} ++ ++static bool arg_type_may_be_refcounted(enum bpf_arg_type type) ++{ ++ return type == ARG_PTR_TO_SOCK_COMMON; ++} ++ ++/* Determine whether the function releases some resources allocated by another ++ * function call. The first reference type argument will be assumed to be ++ * released by release_reference(). ++ */ ++static bool is_release_function(enum bpf_func_id func_id) ++{ ++ return func_id == BPF_FUNC_sk_release; ++} ++ ++static bool is_acquire_function(enum bpf_func_id func_id) ++{ ++ return func_id == BPF_FUNC_sk_lookup_tcp || ++ func_id == BPF_FUNC_sk_lookup_udp || ++ func_id == BPF_FUNC_skc_lookup_tcp; ++} ++ ++static bool is_ptr_cast_function(enum bpf_func_id func_id) ++{ ++ return func_id == BPF_FUNC_tcp_sock || ++ func_id == BPF_FUNC_sk_fullsock; ++} + + /* string representation of 'enum bpf_reg_type' */ + static const char * const reg_type_str[] = { + [NOT_INIT] = "?", +- [UNKNOWN_VALUE] = "inv", ++ [SCALAR_VALUE] = "inv", + [PTR_TO_CTX] = "ctx", + [CONST_PTR_TO_MAP] = "map_ptr", + [PTR_TO_MAP_VALUE] = "map_value", + [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", +- [FRAME_PTR] = "fp", + [PTR_TO_STACK] = "fp", +- [CONST_IMM] = "imm", ++ [PTR_TO_PACKET] = "pkt", ++ [PTR_TO_PACKET_META] = "pkt_meta", ++ [PTR_TO_PACKET_END] = "pkt_end", ++ [PTR_TO_FLOW_KEYS] = "flow_keys", ++ [PTR_TO_SOCKET] = "sock", ++ [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", ++ [PTR_TO_SOCK_COMMON] = "sock_common", ++ [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", ++ [PTR_TO_TCP_SOCK] = "tcp_sock", ++ [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", ++ [PTR_TO_TP_BUFFER] = "tp_buffer", ++ [PTR_TO_XDP_SOCK] = "xdp_sock", + }; + +-static void print_verifier_state(struct verifier_env *env) ++static char slot_type_char[] = { ++ [STACK_INVALID] = '?', ++ [STACK_SPILL] = 'r', ++ [STACK_MISC] = 'm', ++ [STACK_ZERO] = '0', ++}; ++ ++static void print_liveness(struct bpf_verifier_env *env, ++ enum bpf_reg_liveness live) ++{ ++ if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) ++ verbose(env, "_"); ++ if (live & REG_LIVE_READ) ++ verbose(env, "r"); ++ if (live & REG_LIVE_WRITTEN) ++ verbose(env, "w"); ++ if (live & REG_LIVE_DONE) ++ verbose(env, "D"); ++} ++ ++static struct bpf_func_state *func(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg) + { ++ struct bpf_verifier_state *cur = env->cur_state; ++ ++ return cur->frame[reg->frameno]; ++} ++ ++static void print_verifier_state(struct bpf_verifier_env *env, ++ const struct bpf_func_state *state) ++{ ++ const struct bpf_reg_state *reg; + enum bpf_reg_type t; + int i; + ++ if (state->frameno) ++ verbose(env, " frame%d:", state->frameno); + for (i = 0; i < MAX_BPF_REG; i++) { +- t = env->cur_state.regs[i].type; ++ reg = &state->regs[i]; ++ t = reg->type; + if (t == NOT_INIT) + continue; +- verbose(" R%d=%s", i, reg_type_str[t]); +- if (t == CONST_IMM || t == PTR_TO_STACK) +- verbose("%d", env->cur_state.regs[i].imm); +- else if (t == CONST_PTR_TO_MAP || t == PTR_TO_MAP_VALUE || +- t == PTR_TO_MAP_VALUE_OR_NULL) +- verbose("(ks=%d,vs=%d)", +- env->cur_state.regs[i].map_ptr->key_size, +- env->cur_state.regs[i].map_ptr->value_size); +- } +- for (i = 0; i < MAX_BPF_STACK; i += BPF_REG_SIZE) { +- if (env->cur_state.stack_slot_type[i] == STACK_SPILL) +- verbose(" fp%d=%s", -MAX_BPF_STACK + i, +- reg_type_str[env->cur_state.spilled_regs[i / BPF_REG_SIZE].type]); +- } +- verbose("\n"); +-} +- +-static const char *const bpf_class_string[] = { +- [BPF_LD] = "ld", +- [BPF_LDX] = "ldx", +- [BPF_ST] = "st", +- [BPF_STX] = "stx", +- [BPF_ALU] = "alu", +- [BPF_JMP] = "jmp", +- [BPF_RET] = "BUG", +- [BPF_ALU64] = "alu64", +-}; ++ verbose(env, " R%d", i); ++ print_liveness(env, reg->live); ++ verbose(env, "=%s", reg_type_str[t]); ++ if (t == SCALAR_VALUE && reg->precise) ++ verbose(env, "P"); ++ if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && ++ tnum_is_const(reg->var_off)) { ++ /* reg->off should be 0 for SCALAR_VALUE */ ++ verbose(env, "%lld", reg->var_off.value + reg->off); ++ } else { ++ verbose(env, "(id=%d", reg->id); ++ if (reg_type_may_be_refcounted_or_null(t)) ++ verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); ++ if (t != SCALAR_VALUE) ++ verbose(env, ",off=%d", reg->off); ++ if (type_is_pkt_pointer(t)) ++ verbose(env, ",r=%d", reg->range); ++ else if (t == CONST_PTR_TO_MAP || ++ t == PTR_TO_MAP_VALUE || ++ t == PTR_TO_MAP_VALUE_OR_NULL) ++ verbose(env, ",ks=%d,vs=%d", ++ reg->map_ptr->key_size, ++ reg->map_ptr->value_size); ++ if (tnum_is_const(reg->var_off)) { ++ /* Typically an immediate SCALAR_VALUE, but ++ * could be a pointer whose offset is too big ++ * for reg->off ++ */ ++ verbose(env, ",imm=%llx", reg->var_off.value); ++ } else { ++ if (reg->smin_value != reg->umin_value && ++ reg->smin_value != S64_MIN) ++ verbose(env, ",smin_value=%lld", ++ (long long)reg->smin_value); ++ if (reg->smax_value != reg->umax_value && ++ reg->smax_value != S64_MAX) ++ verbose(env, ",smax_value=%lld", ++ (long long)reg->smax_value); ++ if (reg->umin_value != 0) ++ verbose(env, ",umin_value=%llu", ++ (unsigned long long)reg->umin_value); ++ if (reg->umax_value != U64_MAX) ++ verbose(env, ",umax_value=%llu", ++ (unsigned long long)reg->umax_value); ++ if (!tnum_is_unknown(reg->var_off)) { ++ char tn_buf[48]; + +-static const char *const bpf_alu_string[16] = { +- [BPF_ADD >> 4] = "+=", +- [BPF_SUB >> 4] = "-=", +- [BPF_MUL >> 4] = "*=", +- [BPF_DIV >> 4] = "/=", +- [BPF_OR >> 4] = "|=", +- [BPF_AND >> 4] = "&=", +- [BPF_LSH >> 4] = "<<=", +- [BPF_RSH >> 4] = ">>=", +- [BPF_NEG >> 4] = "neg", +- [BPF_MOD >> 4] = "%=", +- [BPF_XOR >> 4] = "^=", +- [BPF_MOV >> 4] = "=", +- [BPF_ARSH >> 4] = "s>>=", +- [BPF_END >> 4] = "endian", +-}; ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, ",var_off=%s", tn_buf); ++ } ++ } ++ verbose(env, ")"); ++ } ++ } ++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { ++ char types_buf[BPF_REG_SIZE + 1]; ++ bool valid = false; ++ int j; ++ ++ for (j = 0; j < BPF_REG_SIZE; j++) { ++ if (state->stack[i].slot_type[j] != STACK_INVALID) ++ valid = true; ++ types_buf[j] = slot_type_char[ ++ state->stack[i].slot_type[j]]; ++ } ++ types_buf[BPF_REG_SIZE] = 0; ++ if (!valid) ++ continue; ++ verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); ++ print_liveness(env, state->stack[i].spilled_ptr.live); ++ if (state->stack[i].slot_type[0] == STACK_SPILL) { ++ reg = &state->stack[i].spilled_ptr; ++ t = reg->type; ++ verbose(env, "=%s", reg_type_str[t]); ++ if (t == SCALAR_VALUE && reg->precise) ++ verbose(env, "P"); ++ if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) ++ verbose(env, "%lld", reg->var_off.value + reg->off); ++ } else { ++ verbose(env, "=%s", types_buf); ++ } ++ } ++ if (state->acquired_refs && state->refs[0].id) { ++ verbose(env, " refs=%d", state->refs[0].id); ++ for (i = 1; i < state->acquired_refs; i++) ++ if (state->refs[i].id) ++ verbose(env, ",%d", state->refs[i].id); ++ } ++ verbose(env, "\n"); ++} + +-static const char *const bpf_ldst_string[] = { +- [BPF_W >> 3] = "u32", +- [BPF_H >> 3] = "u16", +- [BPF_B >> 3] = "u8", +- [BPF_DW >> 3] = "u64", +-}; ++#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ ++static int copy_##NAME##_state(struct bpf_func_state *dst, \ ++ const struct bpf_func_state *src) \ ++{ \ ++ if (!src->FIELD) \ ++ return 0; \ ++ if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ ++ /* internal bug, make state invalid to reject the program */ \ ++ memset(dst, 0, sizeof(*dst)); \ ++ return -EFAULT; \ ++ } \ ++ memcpy(dst->FIELD, src->FIELD, \ ++ sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ ++ return 0; \ ++} ++/* copy_reference_state() */ ++COPY_STATE_FN(reference, acquired_refs, refs, 1) ++/* copy_stack_state() */ ++COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) ++#undef COPY_STATE_FN ++ ++#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ ++static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ ++ bool copy_old) \ ++{ \ ++ u32 old_size = state->COUNT; \ ++ struct bpf_##NAME##_state *new_##FIELD; \ ++ int slot = size / SIZE; \ ++ \ ++ if (size <= old_size || !size) { \ ++ if (copy_old) \ ++ return 0; \ ++ state->COUNT = slot * SIZE; \ ++ if (!size && old_size) { \ ++ kfree(state->FIELD); \ ++ state->FIELD = NULL; \ ++ } \ ++ return 0; \ ++ } \ ++ new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ ++ GFP_KERNEL); \ ++ if (!new_##FIELD) \ ++ return -ENOMEM; \ ++ if (copy_old) { \ ++ if (state->FIELD) \ ++ memcpy(new_##FIELD, state->FIELD, \ ++ sizeof(*new_##FIELD) * (old_size / SIZE)); \ ++ memset(new_##FIELD + old_size / SIZE, 0, \ ++ sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ ++ } \ ++ state->COUNT = slot * SIZE; \ ++ kfree(state->FIELD); \ ++ state->FIELD = new_##FIELD; \ ++ return 0; \ ++} ++/* realloc_reference_state() */ ++REALLOC_STATE_FN(reference, acquired_refs, refs, 1) ++/* realloc_stack_state() */ ++REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) ++#undef REALLOC_STATE_FN ++ ++/* do_check() starts with zero-sized stack in struct bpf_verifier_state to ++ * make it consume minimal amount of memory. check_stack_write() access from ++ * the program calls into realloc_func_state() to grow the stack size. ++ * Note there is a non-zero 'parent' pointer inside bpf_verifier_state ++ * which realloc_stack_state() copies over. It points to previous ++ * bpf_verifier_state which is never reallocated. ++ */ ++static int realloc_func_state(struct bpf_func_state *state, int stack_size, ++ int refs_size, bool copy_old) ++{ ++ int err = realloc_reference_state(state, refs_size, copy_old); ++ if (err) ++ return err; ++ return realloc_stack_state(state, stack_size, copy_old); ++} + +-static const char *const bpf_jmp_string[16] = { +- [BPF_JA >> 4] = "jmp", +- [BPF_JEQ >> 4] = "==", +- [BPF_JGT >> 4] = ">", +- [BPF_JGE >> 4] = ">=", +- [BPF_JSET >> 4] = "&", +- [BPF_JNE >> 4] = "!=", +- [BPF_JSGT >> 4] = "s>", +- [BPF_JSGE >> 4] = "s>=", +- [BPF_CALL >> 4] = "call", +- [BPF_EXIT >> 4] = "exit", +-}; ++/* Acquire a pointer id from the env and update the state->refs to include ++ * this new pointer reference. ++ * On success, returns a valid pointer id to associate with the register ++ * On failure, returns a negative errno. ++ */ ++static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) ++{ ++ struct bpf_func_state *state = cur_func(env); ++ int new_ofs = state->acquired_refs; ++ int id, err; + +-static void print_bpf_insn(const struct verifier_env *env, +- const struct bpf_insn *insn) ++ err = realloc_reference_state(state, state->acquired_refs + 1, true); ++ if (err) ++ return err; ++ id = ++env->id_gen; ++ state->refs[new_ofs].id = id; ++ state->refs[new_ofs].insn_idx = insn_idx; ++ ++ return id; ++} ++ ++/* release function corresponding to acquire_reference_state(). Idempotent. */ ++static int release_reference_state(struct bpf_func_state *state, int ptr_id) + { +- u8 class = BPF_CLASS(insn->code); ++ int i, last_idx; + +- if (class == BPF_ALU || class == BPF_ALU64) { +- if (BPF_SRC(insn->code) == BPF_X) +- verbose("(%02x) %sr%d %s %sr%d\n", +- insn->code, class == BPF_ALU ? "(u32) " : "", +- insn->dst_reg, +- bpf_alu_string[BPF_OP(insn->code) >> 4], +- class == BPF_ALU ? "(u32) " : "", +- insn->src_reg); +- else +- verbose("(%02x) %sr%d %s %s%d\n", +- insn->code, class == BPF_ALU ? "(u32) " : "", +- insn->dst_reg, +- bpf_alu_string[BPF_OP(insn->code) >> 4], +- class == BPF_ALU ? "(u32) " : "", +- insn->imm); +- } else if (class == BPF_STX) { +- if (BPF_MODE(insn->code) == BPF_MEM) +- verbose("(%02x) *(%s *)(r%d %+d) = r%d\n", +- insn->code, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->dst_reg, +- insn->off, insn->src_reg); +- else if (BPF_MODE(insn->code) == BPF_XADD) +- verbose("(%02x) lock *(%s *)(r%d %+d) += r%d\n", +- insn->code, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->dst_reg, insn->off, +- insn->src_reg); +- else +- verbose("BUG_%02x\n", insn->code); +- } else if (class == BPF_ST) { +- if (BPF_MODE(insn->code) != BPF_MEM) { +- verbose("BUG_st_%02x\n", insn->code); +- return; +- } +- verbose("(%02x) *(%s *)(r%d %+d) = %d\n", +- insn->code, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->dst_reg, +- insn->off, insn->imm); +- } else if (class == BPF_LDX) { +- if (BPF_MODE(insn->code) != BPF_MEM) { +- verbose("BUG_ldx_%02x\n", insn->code); +- return; ++ last_idx = state->acquired_refs - 1; ++ for (i = 0; i < state->acquired_refs; i++) { ++ if (state->refs[i].id == ptr_id) { ++ if (last_idx && i != last_idx) ++ memcpy(&state->refs[i], &state->refs[last_idx], ++ sizeof(*state->refs)); ++ memset(&state->refs[last_idx], 0, sizeof(*state->refs)); ++ state->acquired_refs--; ++ return 0; + } +- verbose("(%02x) r%d = *(%s *)(r%d %+d)\n", +- insn->code, insn->dst_reg, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->src_reg, insn->off); +- } else if (class == BPF_LD) { +- if (BPF_MODE(insn->code) == BPF_ABS) { +- verbose("(%02x) r0 = *(%s *)skb[%d]\n", +- insn->code, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->imm); +- } else if (BPF_MODE(insn->code) == BPF_IND) { +- verbose("(%02x) r0 = *(%s *)skb[r%d + %d]\n", +- insn->code, +- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], +- insn->src_reg, insn->imm); +- } else if (BPF_MODE(insn->code) == BPF_IMM && +- BPF_SIZE(insn->code) == BPF_DW) { +- /* At this point, we already made sure that the second +- * part of the ldimm64 insn is accessible. +- */ +- u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; +- bool map_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD; ++ } ++ return -EINVAL; ++} + +- if (map_ptr && !env->allow_ptr_leaks) +- imm = 0; ++static int transfer_reference_state(struct bpf_func_state *dst, ++ struct bpf_func_state *src) ++{ ++ int err = realloc_reference_state(dst, src->acquired_refs, false); ++ if (err) ++ return err; ++ err = copy_reference_state(dst, src); ++ if (err) ++ return err; ++ return 0; ++} + +- verbose("(%02x) r%d = 0x%llx\n", insn->code, +- insn->dst_reg, (unsigned long long)imm); +- } else { +- verbose("BUG_ld_%02x\n", insn->code); +- return; +- } +- } else if (class == BPF_JMP) { +- u8 opcode = BPF_OP(insn->code); ++static void free_func_state(struct bpf_func_state *state) ++{ ++ if (!state) ++ return; ++ kfree(state->refs); ++ kfree(state->stack); ++ kfree(state); ++} + +- if (opcode == BPF_CALL) { +- verbose("(%02x) call %d\n", insn->code, insn->imm); +- } else if (insn->code == (BPF_JMP | BPF_JA)) { +- verbose("(%02x) goto pc%+d\n", +- insn->code, insn->off); +- } else if (insn->code == (BPF_JMP | BPF_EXIT)) { +- verbose("(%02x) exit\n", insn->code); +- } else if (BPF_SRC(insn->code) == BPF_X) { +- verbose("(%02x) if r%d %s r%d goto pc%+d\n", +- insn->code, insn->dst_reg, +- bpf_jmp_string[BPF_OP(insn->code) >> 4], +- insn->src_reg, insn->off); +- } else { +- verbose("(%02x) if r%d %s 0x%x goto pc%+d\n", +- insn->code, insn->dst_reg, +- bpf_jmp_string[BPF_OP(insn->code) >> 4], +- insn->imm, insn->off); ++static void clear_jmp_history(struct bpf_verifier_state *state) ++{ ++ kfree(state->jmp_history); ++ state->jmp_history = NULL; ++ state->jmp_history_cnt = 0; ++} ++ ++static void free_verifier_state(struct bpf_verifier_state *state, ++ bool free_self) ++{ ++ int i; ++ ++ for (i = 0; i <= state->curframe; i++) { ++ free_func_state(state->frame[i]); ++ state->frame[i] = NULL; ++ } ++ clear_jmp_history(state); ++ if (free_self) ++ kfree(state); ++} ++ ++/* copy verifier state from src to dst growing dst stack space ++ * when necessary to accommodate larger src stack ++ */ ++static int copy_func_state(struct bpf_func_state *dst, ++ const struct bpf_func_state *src) ++{ ++ int err; ++ ++ err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, ++ false); ++ if (err) ++ return err; ++ memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); ++ err = copy_reference_state(dst, src); ++ if (err) ++ return err; ++ return copy_stack_state(dst, src); ++} ++ ++static int copy_verifier_state(struct bpf_verifier_state *dst_state, ++ const struct bpf_verifier_state *src) ++{ ++ struct bpf_func_state *dst; ++ u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt; ++ int i, err; ++ ++ if (dst_state->jmp_history_cnt < src->jmp_history_cnt) { ++ kfree(dst_state->jmp_history); ++ dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER); ++ if (!dst_state->jmp_history) ++ return -ENOMEM; ++ } ++ memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz); ++ dst_state->jmp_history_cnt = src->jmp_history_cnt; ++ ++ /* if dst has more stack frames then src frame, free them */ ++ for (i = src->curframe + 1; i <= dst_state->curframe; i++) { ++ free_func_state(dst_state->frame[i]); ++ dst_state->frame[i] = NULL; ++ } ++ dst_state->speculative = src->speculative; ++ dst_state->curframe = src->curframe; ++ dst_state->active_spin_lock = src->active_spin_lock; ++ dst_state->branches = src->branches; ++ dst_state->parent = src->parent; ++ dst_state->first_insn_idx = src->first_insn_idx; ++ dst_state->last_insn_idx = src->last_insn_idx; ++ for (i = 0; i <= src->curframe; i++) { ++ dst = dst_state->frame[i]; ++ if (!dst) { ++ dst = kzalloc(sizeof(*dst), GFP_KERNEL); ++ if (!dst) ++ return -ENOMEM; ++ dst_state->frame[i] = dst; + } +- } else { +- verbose("(%02x) %s\n", insn->code, bpf_class_string[class]); ++ err = copy_func_state(dst, src->frame[i]); ++ if (err) ++ return err; + } ++ return 0; + } + +-static int pop_stack(struct verifier_env *env, int *prev_insn_idx) ++static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) + { +- struct verifier_stack_elem *elem; +- int insn_idx; ++ while (st) { ++ u32 br = --st->branches; ++ ++ /* WARN_ON(br > 1) technically makes sense here, ++ * but see comment in push_stack(), hence: ++ */ ++ WARN_ONCE((int)br < 0, ++ "BUG update_branch_counts:branches_to_explore=%d\n", ++ br); ++ if (br) ++ break; ++ st = st->parent; ++ } ++} ++ ++static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, ++ int *insn_idx) ++{ ++ struct bpf_verifier_state *cur = env->cur_state; ++ struct bpf_verifier_stack_elem *elem, *head = env->head; ++ int err; + + if (env->head == NULL) +- return -1; ++ return -ENOENT; + +- memcpy(&env->cur_state, &env->head->st, sizeof(env->cur_state)); +- insn_idx = env->head->insn_idx; ++ if (cur) { ++ err = copy_verifier_state(cur, &head->st); ++ if (err) ++ return err; ++ } ++ if (insn_idx) ++ *insn_idx = head->insn_idx; + if (prev_insn_idx) +- *prev_insn_idx = env->head->prev_insn_idx; +- elem = env->head->next; +- kfree(env->head); ++ *prev_insn_idx = head->prev_insn_idx; ++ elem = head->next; ++ free_verifier_state(&head->st, false); ++ kfree(head); + env->head = elem; + env->stack_size--; +- return insn_idx; ++ return 0; + } + +-static struct verifier_state *push_stack(struct verifier_env *env, int insn_idx, +- int prev_insn_idx) ++static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, ++ int insn_idx, int prev_insn_idx, ++ bool speculative) + { +- struct verifier_stack_elem *elem; ++ struct bpf_verifier_state *cur = env->cur_state; ++ struct bpf_verifier_stack_elem *elem; ++ int err; + +- elem = kmalloc(sizeof(struct verifier_stack_elem), GFP_KERNEL); ++ elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); + if (!elem) + goto err; + +- memcpy(&elem->st, &env->cur_state, sizeof(env->cur_state)); + elem->insn_idx = insn_idx; + elem->prev_insn_idx = prev_insn_idx; + elem->next = env->head; + env->head = elem; + env->stack_size++; +- if (env->stack_size > 1024) { +- verbose("BPF program is too complex\n"); ++ err = copy_verifier_state(&elem->st, cur); ++ if (err) ++ goto err; ++ elem->st.speculative |= speculative; ++ if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { ++ verbose(env, "The sequence of %d jumps is too complex.\n", ++ env->stack_size); + goto err; + } ++ if (elem->st.parent) { ++ ++elem->st.parent->branches; ++ /* WARN_ON(branches > 2) technically makes sense here, ++ * but ++ * 1. speculative states will bump 'branches' for non-branch ++ * instructions ++ * 2. is_state_visited() heuristics may decide not to create ++ * a new state for a sequence of branches and all such current ++ * and cloned states will be pointing to a single parent state ++ * which might have large 'branches' count. ++ */ ++ } + return &elem->st; + err: ++ free_verifier_state(env->cur_state, true); ++ env->cur_state = NULL; + /* pop all elements and return */ +- while (pop_stack(env, NULL) >= 0); ++ while (!pop_stack(env, NULL, NULL)); + return NULL; + } + +@@ -471,29 +852,225 @@ static const int caller_saved[CALLER_SAV + BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 + }; + +-static void init_reg_state(struct reg_state *regs) ++static void __mark_reg_not_init(const struct bpf_verifier_env *env, ++ struct bpf_reg_state *reg); ++ ++/* Mark the unknown part of a register (variable offset or scalar value) as ++ * known to have the value @imm. ++ */ ++static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) ++{ ++ /* Clear id, off, and union(map_ptr, range) */ ++ memset(((u8 *)reg) + sizeof(reg->type), 0, ++ offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); ++ reg->var_off = tnum_const(imm); ++ reg->smin_value = (s64)imm; ++ reg->smax_value = (s64)imm; ++ reg->umin_value = imm; ++ reg->umax_value = imm; ++} ++ ++/* Mark the 'variable offset' part of a register as zero. This should be ++ * used only on registers holding a pointer type. ++ */ ++static void __mark_reg_known_zero(struct bpf_reg_state *reg) ++{ ++ __mark_reg_known(reg, 0); ++} ++ ++static void __mark_reg_const_zero(struct bpf_reg_state *reg) ++{ ++ __mark_reg_known(reg, 0); ++ reg->type = SCALAR_VALUE; ++} ++ ++static void mark_reg_known_zero(struct bpf_verifier_env *env, ++ struct bpf_reg_state *regs, u32 regno) ++{ ++ if (WARN_ON(regno >= MAX_BPF_REG)) { ++ verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); ++ /* Something bad happened, let's kill all regs */ ++ for (regno = 0; regno < MAX_BPF_REG; regno++) ++ __mark_reg_not_init(env, regs + regno); ++ return; ++ } ++ __mark_reg_known_zero(regs + regno); ++} ++ ++static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) ++{ ++ return type_is_pkt_pointer(reg->type); ++} ++ ++static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) ++{ ++ return reg_is_pkt_pointer(reg) || ++ reg->type == PTR_TO_PACKET_END; ++} ++ ++/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ ++static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, ++ enum bpf_reg_type which) ++{ ++ /* The register can already have a range from prior markings. ++ * This is fine as long as it hasn't been advanced from its ++ * origin. ++ */ ++ return reg->type == which && ++ reg->id == 0 && ++ reg->off == 0 && ++ tnum_equals_const(reg->var_off, 0); ++} ++ ++/* Attempts to improve min/max values based on var_off information */ ++static void __update_reg_bounds(struct bpf_reg_state *reg) ++{ ++ /* min signed is max(sign bit) | min(other bits) */ ++ reg->smin_value = max_t(s64, reg->smin_value, ++ reg->var_off.value | (reg->var_off.mask & S64_MIN)); ++ /* max signed is min(sign bit) | max(other bits) */ ++ reg->smax_value = min_t(s64, reg->smax_value, ++ reg->var_off.value | (reg->var_off.mask & S64_MAX)); ++ reg->umin_value = max(reg->umin_value, reg->var_off.value); ++ reg->umax_value = min(reg->umax_value, ++ reg->var_off.value | reg->var_off.mask); ++} ++ ++/* Uses signed min/max values to inform unsigned, and vice-versa */ ++static void __reg_deduce_bounds(struct bpf_reg_state *reg) + { ++ /* Learn sign from signed bounds. ++ * If we cannot cross the sign boundary, then signed and unsigned bounds ++ * are the same, so combine. This works even in the negative case, e.g. ++ * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. ++ */ ++ if (reg->smin_value >= 0 || reg->smax_value < 0) { ++ reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, ++ reg->umin_value); ++ reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, ++ reg->umax_value); ++ return; ++ } ++ /* Learn sign from unsigned bounds. Signed bounds cross the sign ++ * boundary, so we must be careful. ++ */ ++ if ((s64)reg->umax_value >= 0) { ++ /* Positive. We can't learn anything from the smin, but smax ++ * is positive, hence safe. ++ */ ++ reg->smin_value = reg->umin_value; ++ reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, ++ reg->umax_value); ++ } else if ((s64)reg->umin_value < 0) { ++ /* Negative. We can't learn anything from the smax, but smin ++ * is negative, hence safe. ++ */ ++ reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, ++ reg->umin_value); ++ reg->smax_value = reg->umax_value; ++ } ++} ++ ++/* Attempts to improve var_off based on unsigned min/max information */ ++static void __reg_bound_offset(struct bpf_reg_state *reg) ++{ ++ reg->var_off = tnum_intersect(reg->var_off, ++ tnum_range(reg->umin_value, ++ reg->umax_value)); ++} ++ ++/* Reset the min/max bounds of a register */ ++static void __mark_reg_unbounded(struct bpf_reg_state *reg) ++{ ++ reg->smin_value = S64_MIN; ++ reg->smax_value = S64_MAX; ++ reg->umin_value = 0; ++ reg->umax_value = U64_MAX; ++} ++ ++/* Mark a register as having a completely unknown (scalar) value. */ ++static void __mark_reg_unknown(const struct bpf_verifier_env *env, ++ struct bpf_reg_state *reg) ++{ ++ /* ++ * Clear type, id, off, and union(map_ptr, range) and ++ * padding between 'type' and union ++ */ ++ memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); ++ reg->type = SCALAR_VALUE; ++ reg->var_off = tnum_unknown; ++ reg->frameno = 0; ++ reg->precise = env->subprog_cnt > 1 || !env->allow_ptr_leaks ? ++ true : false; ++ __mark_reg_unbounded(reg); ++} ++ ++static void mark_reg_unknown(struct bpf_verifier_env *env, ++ struct bpf_reg_state *regs, u32 regno) ++{ ++ if (WARN_ON(regno >= MAX_BPF_REG)) { ++ verbose(env, "mark_reg_unknown(regs, %u)\n", regno); ++ /* Something bad happened, let's kill all regs except FP */ ++ for (regno = 0; regno < BPF_REG_FP; regno++) ++ __mark_reg_not_init(env, regs + regno); ++ return; ++ } ++ __mark_reg_unknown(env, regs + regno); ++} ++ ++static void __mark_reg_not_init(const struct bpf_verifier_env *env, ++ struct bpf_reg_state *reg) ++{ ++ __mark_reg_unknown(env, reg); ++ reg->type = NOT_INIT; ++} ++ ++static void mark_reg_not_init(struct bpf_verifier_env *env, ++ struct bpf_reg_state *regs, u32 regno) ++{ ++ if (WARN_ON(regno >= MAX_BPF_REG)) { ++ verbose(env, "mark_reg_not_init(regs, %u)\n", regno); ++ /* Something bad happened, let's kill all regs except FP */ ++ for (regno = 0; regno < BPF_REG_FP; regno++) ++ __mark_reg_not_init(env, regs + regno); ++ return; ++ } ++ __mark_reg_not_init(env, regs + regno); ++} ++ ++#define DEF_NOT_SUBREG (0) ++static void init_reg_state(struct bpf_verifier_env *env, ++ struct bpf_func_state *state) ++{ ++ struct bpf_reg_state *regs = state->regs; + int i; + + for (i = 0; i < MAX_BPF_REG; i++) { +- regs[i].type = NOT_INIT; +- regs[i].imm = 0; +- regs[i].map_ptr = NULL; ++ mark_reg_not_init(env, regs, i); ++ regs[i].live = REG_LIVE_NONE; ++ regs[i].parent = NULL; ++ regs[i].subreg_def = DEF_NOT_SUBREG; + } + + /* frame pointer */ +- regs[BPF_REG_FP].type = FRAME_PTR; ++ regs[BPF_REG_FP].type = PTR_TO_STACK; ++ mark_reg_known_zero(env, regs, BPF_REG_FP); ++ regs[BPF_REG_FP].frameno = state->frameno; + + /* 1st arg to a function */ + regs[BPF_REG_1].type = PTR_TO_CTX; ++ mark_reg_known_zero(env, regs, BPF_REG_1); + } + +-static void mark_reg_unknown_value(struct reg_state *regs, u32 regno) ++#define BPF_MAIN_FUNC (-1) ++static void init_func_state(struct bpf_verifier_env *env, ++ struct bpf_func_state *state, ++ int callsite, int frameno, int subprogno) + { +- BUG_ON(regno >= MAX_BPF_REG); +- regs[regno].type = UNKNOWN_VALUE; +- regs[regno].imm = 0; +- regs[regno].map_ptr = NULL; ++ state->callsite = callsite; ++ state->frameno = frameno; ++ state->subprogno = subprogno; ++ init_reg_state(env, state); + } + + enum reg_arg_type { +@@ -502,44 +1079,760 @@ enum reg_arg_type { + DST_OP_NO_MARK /* same as above, check only, don't mark */ + }; + +-static int check_reg_arg(struct reg_state *regs, u32 regno, ++static int cmp_subprogs(const void *a, const void *b) ++{ ++ return ((struct bpf_subprog_info *)a)->start - ++ ((struct bpf_subprog_info *)b)->start; ++} ++ ++static int find_subprog(struct bpf_verifier_env *env, int off) ++{ ++ struct bpf_subprog_info *p; ++ ++ p = bsearch(&off, env->subprog_info, env->subprog_cnt, ++ sizeof(env->subprog_info[0]), cmp_subprogs); ++ if (!p) ++ return -ENOENT; ++ return p - env->subprog_info; ++ ++} ++ ++static int add_subprog(struct bpf_verifier_env *env, int off) ++{ ++ int insn_cnt = env->prog->len; ++ int ret; ++ ++ if (off >= insn_cnt || off < 0) { ++ verbose(env, "call to invalid destination\n"); ++ return -EINVAL; ++ } ++ ret = find_subprog(env, off); ++ if (ret >= 0) ++ return 0; ++ if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { ++ verbose(env, "too many subprograms\n"); ++ return -E2BIG; ++ } ++ env->subprog_info[env->subprog_cnt++].start = off; ++ sort(env->subprog_info, env->subprog_cnt, ++ sizeof(env->subprog_info[0]), cmp_subprogs, NULL); ++ return 0; ++} ++ ++static int check_subprogs(struct bpf_verifier_env *env) ++{ ++ int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; ++ struct bpf_subprog_info *subprog = env->subprog_info; ++ struct bpf_insn *insn = env->prog->insnsi; ++ int insn_cnt = env->prog->len; ++ ++ /* Add entry function. */ ++ ret = add_subprog(env, 0); ++ if (ret < 0) ++ return ret; ++ ++ /* determine subprog starts. The end is one before the next starts */ ++ for (i = 0; i < insn_cnt; i++) { ++ if (insn[i].code != (BPF_JMP | BPF_CALL)) ++ continue; ++ if (insn[i].src_reg != BPF_PSEUDO_CALL) ++ continue; ++ if (!env->allow_ptr_leaks) { ++ verbose(env, "function calls to other bpf functions are allowed for root only\n"); ++ return -EPERM; ++ } ++ ret = add_subprog(env, i + insn[i].imm + 1); ++ if (ret < 0) ++ return ret; ++ } ++ ++ /* Add a fake 'exit' subprog which could simplify subprog iteration ++ * logic. 'subprog_cnt' should not be increased. ++ */ ++ subprog[env->subprog_cnt].start = insn_cnt; ++ ++ if (env->log.level & BPF_LOG_LEVEL2) ++ for (i = 0; i < env->subprog_cnt; i++) ++ verbose(env, "func#%d @%d\n", i, subprog[i].start); ++ ++ /* now check that all jumps are within the same subprog */ ++ subprog_start = subprog[cur_subprog].start; ++ subprog_end = subprog[cur_subprog + 1].start; ++ for (i = 0; i < insn_cnt; i++) { ++ u8 code = insn[i].code; ++ ++ if (code == (BPF_JMP | BPF_CALL) && ++ insn[i].imm == BPF_FUNC_tail_call && ++ insn[i].src_reg != BPF_PSEUDO_CALL) ++ subprog[cur_subprog].has_tail_call = true; ++ if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) ++ goto next; ++ if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) ++ goto next; ++ off = i + insn[i].off + 1; ++ if (off < subprog_start || off >= subprog_end) { ++ verbose(env, "jump out of range from insn %d to %d\n", i, off); ++ return -EINVAL; ++ } ++next: ++ if (i == subprog_end - 1) { ++ /* to avoid fall-through from one subprog into another ++ * the last insn of the subprog should be either exit ++ * or unconditional jump back ++ */ ++ if (code != (BPF_JMP | BPF_EXIT) && ++ code != (BPF_JMP | BPF_JA)) { ++ verbose(env, "last insn is not an exit or jmp\n"); ++ return -EINVAL; ++ } ++ subprog_start = subprog_end; ++ cur_subprog++; ++ if (cur_subprog < env->subprog_cnt) ++ subprog_end = subprog[cur_subprog + 1].start; ++ } ++ } ++ return 0; ++} ++ ++/* Parentage chain of this register (or stack slot) should take care of all ++ * issues like callee-saved registers, stack slot allocation time, etc. ++ */ ++static int mark_reg_read(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *state, ++ struct bpf_reg_state *parent, u8 flag) ++{ ++ bool writes = parent == state->parent; /* Observe write marks */ ++ int cnt = 0; ++ ++ while (parent) { ++ /* if read wasn't screened by an earlier write ... */ ++ if (writes && state->live & REG_LIVE_WRITTEN) ++ break; ++ if (parent->live & REG_LIVE_DONE) { ++ verbose(env, "verifier BUG type %s var_off %lld off %d\n", ++ reg_type_str[parent->type], ++ parent->var_off.value, parent->off); ++ return -EFAULT; ++ } ++ /* The first condition is more likely to be true than the ++ * second, checked it first. ++ */ ++ if ((parent->live & REG_LIVE_READ) == flag || ++ parent->live & REG_LIVE_READ64) ++ /* The parentage chain never changes and ++ * this parent was already marked as LIVE_READ. ++ * There is no need to keep walking the chain again and ++ * keep re-marking all parents as LIVE_READ. ++ * This case happens when the same register is read ++ * multiple times without writes into it in-between. ++ * Also, if parent has the stronger REG_LIVE_READ64 set, ++ * then no need to set the weak REG_LIVE_READ32. ++ */ ++ break; ++ /* ... then we depend on parent's value */ ++ parent->live |= flag; ++ /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ ++ if (flag == REG_LIVE_READ64) ++ parent->live &= ~REG_LIVE_READ32; ++ state = parent; ++ parent = state->parent; ++ writes = true; ++ cnt++; ++ } ++ ++ if (env->longest_mark_read_walk < cnt) ++ env->longest_mark_read_walk = cnt; ++ return 0; ++} ++ ++/* This function is supposed to be used by the following 32-bit optimization ++ * code only. It returns TRUE if the source or destination register operates ++ * on 64-bit, otherwise return FALSE. ++ */ ++static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, ++ u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) ++{ ++ u8 code, class, op; ++ ++ code = insn->code; ++ class = BPF_CLASS(code); ++ op = BPF_OP(code); ++ if (class == BPF_JMP) { ++ /* BPF_EXIT for "main" will reach here. Return TRUE ++ * conservatively. ++ */ ++ if (op == BPF_EXIT) ++ return true; ++ if (op == BPF_CALL) { ++ /* BPF to BPF call will reach here because of marking ++ * caller saved clobber with DST_OP_NO_MARK for which we ++ * don't care the register def because they are anyway ++ * marked as NOT_INIT already. ++ */ ++ if (insn->src_reg == BPF_PSEUDO_CALL) ++ return false; ++ /* Helper call will reach here because of arg type ++ * check, conservatively return TRUE. ++ */ ++ if (t == SRC_OP) ++ return true; ++ ++ return false; ++ } ++ } ++ ++ if (class == BPF_ALU64 || class == BPF_JMP || ++ /* BPF_END always use BPF_ALU class. */ ++ (class == BPF_ALU && op == BPF_END && insn->imm == 64)) ++ return true; ++ ++ if (class == BPF_ALU || class == BPF_JMP32) ++ return false; ++ ++ if (class == BPF_LDX) { ++ if (t != SRC_OP) ++ return BPF_SIZE(code) == BPF_DW; ++ /* LDX source must be ptr. */ ++ return true; ++ } ++ ++ if (class == BPF_STX) { ++ if (reg->type != SCALAR_VALUE) ++ return true; ++ return BPF_SIZE(code) == BPF_DW; ++ } ++ ++ if (class == BPF_LD) { ++ u8 mode = BPF_MODE(code); ++ ++ /* LD_IMM64 */ ++ if (mode == BPF_IMM) ++ return true; ++ ++ /* Both LD_IND and LD_ABS return 32-bit data. */ ++ if (t != SRC_OP) ++ return false; ++ ++ /* Implicit ctx ptr. */ ++ if (regno == BPF_REG_6) ++ return true; ++ ++ /* Explicit source could be any width. */ ++ return true; ++ } ++ ++ if (class == BPF_ST) ++ /* The only source register for BPF_ST is a ptr. */ ++ return true; ++ ++ /* Conservatively return true at default. */ ++ return true; ++} ++ ++/* Return TRUE if INSN doesn't have explicit value define. */ ++static bool insn_no_def(struct bpf_insn *insn) ++{ ++ u8 class = BPF_CLASS(insn->code); ++ ++ return (class == BPF_JMP || class == BPF_JMP32 || ++ class == BPF_STX || class == BPF_ST); ++} ++ ++/* Return TRUE if INSN has defined any 32-bit value explicitly. */ ++static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) ++{ ++ if (insn_no_def(insn)) ++ return false; ++ ++ return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP); ++} ++ ++static void mark_insn_zext(struct bpf_verifier_env *env, ++ struct bpf_reg_state *reg) ++{ ++ s32 def_idx = reg->subreg_def; ++ ++ if (def_idx == DEF_NOT_SUBREG) ++ return; ++ ++ env->insn_aux_data[def_idx - 1].zext_dst = true; ++ /* The dst will be zero extended, so won't be sub-register anymore. */ ++ reg->subreg_def = DEF_NOT_SUBREG; ++} ++ ++static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, + enum reg_arg_type t) + { ++ struct bpf_verifier_state *vstate = env->cur_state; ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; ++ struct bpf_reg_state *reg, *regs = state->regs; ++ bool rw64; ++ + if (regno >= MAX_BPF_REG) { +- verbose("R%d is invalid\n", regno); ++ verbose(env, "R%d is invalid\n", regno); + return -EINVAL; + } + ++ reg = ®s[regno]; ++ rw64 = is_reg64(env, insn, regno, reg, t); + if (t == SRC_OP) { + /* check whether register used as source operand can be read */ +- if (regs[regno].type == NOT_INIT) { +- verbose("R%d !read_ok\n", regno); ++ if (reg->type == NOT_INIT) { ++ verbose(env, "R%d !read_ok\n", regno); + return -EACCES; + } ++ /* We don't need to worry about FP liveness because it's read-only */ ++ if (regno == BPF_REG_FP) ++ return 0; ++ ++ if (rw64) ++ mark_insn_zext(env, reg); ++ ++ return mark_reg_read(env, reg, reg->parent, ++ rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); + } else { + /* check whether register used as dest operand can be written to */ + if (regno == BPF_REG_FP) { +- verbose("frame pointer is read only\n"); ++ verbose(env, "frame pointer is read only\n"); + return -EACCES; + } ++ reg->live |= REG_LIVE_WRITTEN; ++ reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; + if (t == DST_OP) +- mark_reg_unknown_value(regs, regno); ++ mark_reg_unknown(env, regs, regno); + } + return 0; + } + +-static int bpf_size_to_bytes(int bpf_size) ++/* for any branch, call, exit record the history of jmps in the given state */ ++static int push_jmp_history(struct bpf_verifier_env *env, ++ struct bpf_verifier_state *cur) + { +- if (bpf_size == BPF_W) +- return 4; +- else if (bpf_size == BPF_H) +- return 2; +- else if (bpf_size == BPF_B) +- return 1; +- else if (bpf_size == BPF_DW) +- return 8; +- else +- return -EINVAL; ++ u32 cnt = cur->jmp_history_cnt; ++ struct bpf_idx_pair *p; ++ ++ cnt++; ++ p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); ++ if (!p) ++ return -ENOMEM; ++ p[cnt - 1].idx = env->insn_idx; ++ p[cnt - 1].prev_idx = env->prev_insn_idx; ++ cur->jmp_history = p; ++ cur->jmp_history_cnt = cnt; ++ return 0; ++} ++ ++/* Backtrack one insn at a time. If idx is not at the top of recorded ++ * history then previous instruction came from straight line execution. ++ */ ++static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, ++ u32 *history) ++{ ++ u32 cnt = *history; ++ ++ if (cnt && st->jmp_history[cnt - 1].idx == i) { ++ i = st->jmp_history[cnt - 1].prev_idx; ++ (*history)--; ++ } else { ++ i--; ++ } ++ return i; ++} ++ ++/* For given verifier state backtrack_insn() is called from the last insn to ++ * the first insn. Its purpose is to compute a bitmask of registers and ++ * stack slots that needs precision in the parent verifier state. ++ */ ++static int backtrack_insn(struct bpf_verifier_env *env, int idx, ++ u32 *reg_mask, u64 *stack_mask) ++{ ++ const struct bpf_insn_cbs cbs = { ++ .cb_print = verbose, ++ .private_data = env, ++ }; ++ struct bpf_insn *insn = env->prog->insnsi + idx; ++ u8 class = BPF_CLASS(insn->code); ++ u8 opcode = BPF_OP(insn->code); ++ u8 mode = BPF_MODE(insn->code); ++ u32 dreg = 1u << insn->dst_reg; ++ u32 sreg = 1u << insn->src_reg; ++ u32 spi; ++ ++ if (insn->code == 0) ++ return 0; ++ if (env->log.level & BPF_LOG_LEVEL) { ++ verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); ++ verbose(env, "%d: ", idx); ++ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); ++ } ++ ++ if (class == BPF_ALU || class == BPF_ALU64) { ++ if (!(*reg_mask & dreg)) ++ return 0; ++ if (opcode == BPF_MOV) { ++ if (BPF_SRC(insn->code) == BPF_X) { ++ /* dreg = sreg ++ * dreg needs precision after this insn ++ * sreg needs precision before this insn ++ */ ++ *reg_mask &= ~dreg; ++ *reg_mask |= sreg; ++ } else { ++ /* dreg = K ++ * dreg needs precision after this insn. ++ * Corresponding register is already marked ++ * as precise=true in this verifier state. ++ * No further markings in parent are necessary ++ */ ++ *reg_mask &= ~dreg; ++ } ++ } else { ++ if (BPF_SRC(insn->code) == BPF_X) { ++ /* dreg += sreg ++ * both dreg and sreg need precision ++ * before this insn ++ */ ++ *reg_mask |= sreg; ++ } /* else dreg += K ++ * dreg still needs precision before this insn ++ */ ++ } ++ } else if (class == BPF_LDX) { ++ if (!(*reg_mask & dreg)) ++ return 0; ++ *reg_mask &= ~dreg; ++ ++ /* scalars can only be spilled into stack w/o losing precision. ++ * Load from any other memory can be zero extended. ++ * The desire to keep that precision is already indicated ++ * by 'precise' mark in corresponding register of this state. ++ * No further tracking necessary. ++ */ ++ if (insn->src_reg != BPF_REG_FP) ++ return 0; ++ if (BPF_SIZE(insn->code) != BPF_DW) ++ return 0; ++ ++ /* dreg = *(u64 *)[fp - off] was a fill from the stack. ++ * that [fp - off] slot contains scalar that needs to be ++ * tracked with precision ++ */ ++ spi = (-insn->off - 1) / BPF_REG_SIZE; ++ if (spi >= 64) { ++ verbose(env, "BUG spi %d\n", spi); ++ WARN_ONCE(1, "verifier backtracking bug"); ++ return -EFAULT; ++ } ++ *stack_mask |= 1ull << spi; ++ } else if (class == BPF_STX || class == BPF_ST) { ++ if (*reg_mask & dreg) ++ /* stx & st shouldn't be using _scalar_ dst_reg ++ * to access memory. It means backtracking ++ * encountered a case of pointer subtraction. ++ */ ++ return -ENOTSUPP; ++ /* scalars can only be spilled into stack */ ++ if (insn->dst_reg != BPF_REG_FP) ++ return 0; ++ if (BPF_SIZE(insn->code) != BPF_DW) ++ return 0; ++ spi = (-insn->off - 1) / BPF_REG_SIZE; ++ if (spi >= 64) { ++ verbose(env, "BUG spi %d\n", spi); ++ WARN_ONCE(1, "verifier backtracking bug"); ++ return -EFAULT; ++ } ++ if (!(*stack_mask & (1ull << spi))) ++ return 0; ++ *stack_mask &= ~(1ull << spi); ++ if (class == BPF_STX) ++ *reg_mask |= sreg; ++ } else if (class == BPF_JMP || class == BPF_JMP32) { ++ if (opcode == BPF_CALL) { ++ if (insn->src_reg == BPF_PSEUDO_CALL) ++ return -ENOTSUPP; ++ /* regular helper call sets R0 */ ++ *reg_mask &= ~1; ++ if (*reg_mask & 0x3f) { ++ /* if backtracing was looking for registers R1-R5 ++ * they should have been found already. ++ */ ++ verbose(env, "BUG regs %x\n", *reg_mask); ++ WARN_ONCE(1, "verifier backtracking bug"); ++ return -EFAULT; ++ } ++ } else if (opcode == BPF_EXIT) { ++ return -ENOTSUPP; ++ } ++ } else if (class == BPF_LD) { ++ if (!(*reg_mask & dreg)) ++ return 0; ++ *reg_mask &= ~dreg; ++ /* It's ld_imm64 or ld_abs or ld_ind. ++ * For ld_imm64 no further tracking of precision ++ * into parent is necessary ++ */ ++ if (mode == BPF_IND || mode == BPF_ABS) ++ /* to be analyzed */ ++ return -ENOTSUPP; ++ } ++ return 0; ++} ++ ++/* the scalar precision tracking algorithm: ++ * . at the start all registers have precise=false. ++ * . scalar ranges are tracked as normal through alu and jmp insns. ++ * . once precise value of the scalar register is used in: ++ * . ptr + scalar alu ++ * . if (scalar cond K|scalar) ++ * . helper_call(.., scalar, ...) where ARG_CONST is expected ++ * backtrack through the verifier states and mark all registers and ++ * stack slots with spilled constants that these scalar regisers ++ * should be precise. ++ * . during state pruning two registers (or spilled stack slots) ++ * are equivalent if both are not precise. ++ * ++ * Note the verifier cannot simply walk register parentage chain, ++ * since many different registers and stack slots could have been ++ * used to compute single precise scalar. ++ * ++ * The approach of starting with precise=true for all registers and then ++ * backtrack to mark a register as not precise when the verifier detects ++ * that program doesn't care about specific value (e.g., when helper ++ * takes register as ARG_ANYTHING parameter) is not safe. ++ * ++ * It's ok to walk single parentage chain of the verifier states. ++ * It's possible that this backtracking will go all the way till 1st insn. ++ * All other branches will be explored for needing precision later. ++ * ++ * The backtracking needs to deal with cases like: ++ * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) ++ * r9 -= r8 ++ * r5 = r9 ++ * if r5 > 0x79f goto pc+7 ++ * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) ++ * r5 += 1 ++ * ... ++ * call bpf_perf_event_output#25 ++ * where .arg5_type = ARG_CONST_SIZE_OR_ZERO ++ * ++ * and this case: ++ * r6 = 1 ++ * call foo // uses callee's r6 inside to compute r0 ++ * r0 += r6 ++ * if r0 == 0 goto ++ * ++ * to track above reg_mask/stack_mask needs to be independent for each frame. ++ * ++ * Also if parent's curframe > frame where backtracking started, ++ * the verifier need to mark registers in both frames, otherwise callees ++ * may incorrectly prune callers. This is similar to ++ * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") ++ * ++ * For now backtracking falls back into conservative marking. ++ */ ++static void mark_all_scalars_precise(struct bpf_verifier_env *env, ++ struct bpf_verifier_state *st) ++{ ++ struct bpf_func_state *func; ++ struct bpf_reg_state *reg; ++ int i, j; ++ ++ /* big hammer: mark all scalars precise in this path. ++ * pop_stack may still get !precise scalars. ++ */ ++ for (; st; st = st->parent) ++ for (i = 0; i <= st->curframe; i++) { ++ func = st->frame[i]; ++ for (j = 0; j < BPF_REG_FP; j++) { ++ reg = &func->regs[j]; ++ if (reg->type != SCALAR_VALUE) ++ continue; ++ reg->precise = true; ++ } ++ for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { ++ if (func->stack[j].slot_type[0] != STACK_SPILL) ++ continue; ++ reg = &func->stack[j].spilled_ptr; ++ if (reg->type != SCALAR_VALUE) ++ continue; ++ reg->precise = true; ++ } ++ } ++} ++ ++static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, ++ int spi) ++{ ++ struct bpf_verifier_state *st = env->cur_state; ++ int first_idx = st->first_insn_idx; ++ int last_idx = env->insn_idx; ++ struct bpf_func_state *func; ++ struct bpf_reg_state *reg; ++ u32 reg_mask = regno >= 0 ? 1u << regno : 0; ++ u64 stack_mask = spi >= 0 ? 1ull << spi : 0; ++ bool skip_first = true; ++ bool new_marks = false; ++ int i, err; ++ ++ if (!env->allow_ptr_leaks) ++ /* backtracking is root only for now */ ++ return 0; ++ ++ func = st->frame[st->curframe]; ++ if (regno >= 0) { ++ reg = &func->regs[regno]; ++ if (reg->type != SCALAR_VALUE) { ++ WARN_ONCE(1, "backtracing misuse"); ++ return -EFAULT; ++ } ++ if (!reg->precise) ++ new_marks = true; ++ else ++ reg_mask = 0; ++ reg->precise = true; ++ } ++ ++ while (spi >= 0) { ++ if (func->stack[spi].slot_type[0] != STACK_SPILL) { ++ stack_mask = 0; ++ break; ++ } ++ reg = &func->stack[spi].spilled_ptr; ++ if (reg->type != SCALAR_VALUE) { ++ stack_mask = 0; ++ break; ++ } ++ if (!reg->precise) ++ new_marks = true; ++ else ++ stack_mask = 0; ++ reg->precise = true; ++ break; ++ } ++ ++ if (!new_marks) ++ return 0; ++ if (!reg_mask && !stack_mask) ++ return 0; ++ for (;;) { ++ DECLARE_BITMAP(mask, 64); ++ u32 history = st->jmp_history_cnt; ++ ++ if (env->log.level & BPF_LOG_LEVEL) ++ verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); ++ for (i = last_idx;;) { ++ if (skip_first) { ++ err = 0; ++ skip_first = false; ++ } else { ++ err = backtrack_insn(env, i, ®_mask, &stack_mask); ++ } ++ if (err == -ENOTSUPP) { ++ mark_all_scalars_precise(env, st); ++ return 0; ++ } else if (err) { ++ return err; ++ } ++ if (!reg_mask && !stack_mask) ++ /* Found assignment(s) into tracked register in this state. ++ * Since this state is already marked, just return. ++ * Nothing to be tracked further in the parent state. ++ */ ++ return 0; ++ if (i == first_idx) ++ break; ++ i = get_prev_insn_idx(st, i, &history); ++ if (i >= env->prog->len) { ++ /* This can happen if backtracking reached insn 0 ++ * and there are still reg_mask or stack_mask ++ * to backtrack. ++ * It means the backtracking missed the spot where ++ * particular register was initialized with a constant. ++ */ ++ verbose(env, "BUG backtracking idx %d\n", i); ++ WARN_ONCE(1, "verifier backtracking bug"); ++ return -EFAULT; ++ } ++ } ++ st = st->parent; ++ if (!st) ++ break; ++ ++ new_marks = false; ++ func = st->frame[st->curframe]; ++ bitmap_from_u64(mask, reg_mask); ++ for_each_set_bit(i, mask, 32) { ++ reg = &func->regs[i]; ++ if (reg->type != SCALAR_VALUE) { ++ reg_mask &= ~(1u << i); ++ continue; ++ } ++ if (!reg->precise) ++ new_marks = true; ++ reg->precise = true; ++ } ++ ++ bitmap_from_u64(mask, stack_mask); ++ for_each_set_bit(i, mask, 64) { ++ if (i >= func->allocated_stack / BPF_REG_SIZE) { ++ /* the sequence of instructions: ++ * 2: (bf) r3 = r10 ++ * 3: (7b) *(u64 *)(r3 -8) = r0 ++ * 4: (79) r4 = *(u64 *)(r10 -8) ++ * doesn't contain jmps. It's backtracked ++ * as a single block. ++ * During backtracking insn 3 is not recognized as ++ * stack access, so at the end of backtracking ++ * stack slot fp-8 is still marked in stack_mask. ++ * However the parent state may not have accessed ++ * fp-8 and it's "unallocated" stack space. ++ * In such case fallback to conservative. ++ */ ++ mark_all_scalars_precise(env, st); ++ return 0; ++ } ++ ++ if (func->stack[i].slot_type[0] != STACK_SPILL) { ++ stack_mask &= ~(1ull << i); ++ continue; ++ } ++ reg = &func->stack[i].spilled_ptr; ++ if (reg->type != SCALAR_VALUE) { ++ stack_mask &= ~(1ull << i); ++ continue; ++ } ++ if (!reg->precise) ++ new_marks = true; ++ reg->precise = true; ++ } ++ if (env->log.level & BPF_LOG_LEVEL) { ++ print_verifier_state(env, func); ++ verbose(env, "parent %s regs=%x stack=%llx marks\n", ++ new_marks ? "didn't have" : "already had", ++ reg_mask, stack_mask); ++ } ++ ++ if (!reg_mask && !stack_mask) ++ break; ++ if (!new_marks) ++ break; ++ ++ last_idx = st->last_insn_idx; ++ first_idx = st->first_insn_idx; ++ } ++ return 0; ++} ++ ++static int mark_chain_precision(struct bpf_verifier_env *env, int regno) ++{ ++ return __mark_chain_precision(env, regno, -1); ++} ++ ++static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) ++{ ++ return __mark_chain_precision(env, -1, spi); + } + + static bool is_spillable_regtype(enum bpf_reg_type type) +@@ -549,129 +1842,932 @@ static bool is_spillable_regtype(enum bp + case PTR_TO_MAP_VALUE_OR_NULL: + case PTR_TO_STACK: + case PTR_TO_CTX: +- case FRAME_PTR: ++ case PTR_TO_PACKET: ++ case PTR_TO_PACKET_META: ++ case PTR_TO_PACKET_END: ++ case PTR_TO_FLOW_KEYS: + case CONST_PTR_TO_MAP: ++ case PTR_TO_SOCKET: ++ case PTR_TO_SOCKET_OR_NULL: ++ case PTR_TO_SOCK_COMMON: ++ case PTR_TO_SOCK_COMMON_OR_NULL: ++ case PTR_TO_TCP_SOCK: ++ case PTR_TO_TCP_SOCK_OR_NULL: ++ case PTR_TO_XDP_SOCK: + return true; + default: + return false; + } + } + ++/* Does this register contain a constant zero? */ ++static bool register_is_null(struct bpf_reg_state *reg) ++{ ++ return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); ++} ++ ++static bool register_is_const(struct bpf_reg_state *reg) ++{ ++ return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); ++} ++ ++static bool __is_pointer_value(bool allow_ptr_leaks, ++ const struct bpf_reg_state *reg) ++{ ++ if (allow_ptr_leaks) ++ return false; ++ ++ return reg->type != SCALAR_VALUE; ++} ++ ++static void save_register_state(struct bpf_func_state *state, ++ int spi, struct bpf_reg_state *reg) ++{ ++ int i; ++ ++ state->stack[spi].spilled_ptr = *reg; ++ state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; ++ ++ for (i = 0; i < BPF_REG_SIZE; i++) ++ state->stack[spi].slot_type[i] = STACK_SPILL; ++} ++ + /* check_stack_read/write functions track spill/fill of registers, + * stack boundary and alignment are checked in check_mem_access() + */ +-static int check_stack_write(struct verifier_state *state, int off, int size, +- int value_regno) ++static int check_stack_write(struct bpf_verifier_env *env, ++ struct bpf_func_state *state, /* func where register points to */ ++ int off, int size, int value_regno, int insn_idx) + { +- int i; ++ struct bpf_func_state *cur; /* state of the current function */ ++ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; ++ u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; ++ struct bpf_reg_state *reg = NULL; ++ ++ err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), ++ state->acquired_refs, true); ++ if (err) ++ return err; + /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, + * so it's aligned access and [off, off + size) are within stack limits + */ ++ if (!env->allow_ptr_leaks && ++ state->stack[spi].slot_type[0] == STACK_SPILL && ++ size != BPF_REG_SIZE) { ++ verbose(env, "attempt to corrupt spilled pointer on stack\n"); ++ return -EACCES; ++ } + +- if (value_regno >= 0 && +- is_spillable_regtype(state->regs[value_regno].type)) { +- ++ cur = env->cur_state->frame[env->cur_state->curframe]; ++ if (value_regno >= 0) ++ reg = &cur->regs[value_regno]; ++ ++ if (reg && size == BPF_REG_SIZE && register_is_const(reg) && ++ !register_is_null(reg) && env->allow_ptr_leaks) { ++ if (dst_reg != BPF_REG_FP) { ++ /* The backtracking logic can only recognize explicit ++ * stack slot address like [fp - 8]. Other spill of ++ * scalar via different register has to be conervative. ++ * Backtrack from here and mark all registers as precise ++ * that contributed into 'reg' being a constant. ++ */ ++ err = mark_chain_precision(env, value_regno); ++ if (err) ++ return err; ++ } ++ save_register_state(state, spi, reg); ++ } else if (reg && is_spillable_regtype(reg->type)) { + /* register containing pointer is being spilled into stack */ + if (size != BPF_REG_SIZE) { +- verbose("invalid size of register spill\n"); ++ verbose_linfo(env, insn_idx, "; "); ++ verbose(env, "invalid size of register spill\n"); + return -EACCES; + } + +- /* save register state */ +- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE] = +- state->regs[value_regno]; ++ if (state != cur && reg->type == PTR_TO_STACK) { ++ verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); ++ return -EINVAL; ++ } + +- for (i = 0; i < BPF_REG_SIZE; i++) +- state->stack_slot_type[MAX_BPF_STACK + off + i] = STACK_SPILL; ++ if (!env->allow_ptr_leaks) { ++ bool sanitize = false; ++ ++ if (state->stack[spi].slot_type[0] == STACK_SPILL && ++ register_is_const(&state->stack[spi].spilled_ptr)) ++ sanitize = true; ++ for (i = 0; i < BPF_REG_SIZE; i++) ++ if (state->stack[spi].slot_type[i] == STACK_MISC) { ++ sanitize = true; ++ break; ++ } ++ if (sanitize) { ++ int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; ++ int soff = (-spi - 1) * BPF_REG_SIZE; ++ ++ /* detected reuse of integer stack slot with a pointer ++ * which means either llvm is reusing stack slot or ++ * an attacker is trying to exploit CVE-2018-3639 ++ * (speculative store bypass) ++ * Have to sanitize that slot with preemptive ++ * store of zero. ++ */ ++ if (*poff && *poff != soff) { ++ /* disallow programs where single insn stores ++ * into two different stack slots, since verifier ++ * cannot sanitize them ++ */ ++ verbose(env, ++ "insn %d cannot access two stack slots fp%d and fp%d", ++ insn_idx, *poff, soff); ++ return -EINVAL; ++ } ++ *poff = soff; ++ } ++ } ++ save_register_state(state, spi, reg); + } else { +- /* regular write of data into stack */ +- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE] = +- (struct reg_state) {}; ++ u8 type = STACK_MISC; ++ ++ /* regular write of data into stack destroys any spilled ptr */ ++ state->stack[spi].spilled_ptr.type = NOT_INIT; ++ /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ ++ if (state->stack[spi].slot_type[0] == STACK_SPILL) ++ for (i = 0; i < BPF_REG_SIZE; i++) ++ state->stack[spi].slot_type[i] = STACK_MISC; ++ ++ /* only mark the slot as written if all 8 bytes were written ++ * otherwise read propagation may incorrectly stop too soon ++ * when stack slots are partially written. ++ * This heuristic means that read propagation will be ++ * conservative, since it will add reg_live_read marks ++ * to stack slots all the way to first state when programs ++ * writes+reads less than 8 bytes ++ */ ++ if (size == BPF_REG_SIZE) ++ state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; ++ ++ /* when we zero initialize stack slots mark them as such */ ++ if (reg && register_is_null(reg)) { ++ /* backtracking doesn't work for STACK_ZERO yet. */ ++ err = mark_chain_precision(env, value_regno); ++ if (err) ++ return err; ++ type = STACK_ZERO; ++ } + ++ /* Mark slots affected by this stack write. */ + for (i = 0; i < size; i++) +- state->stack_slot_type[MAX_BPF_STACK + off + i] = STACK_MISC; ++ state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = ++ type; + } + return 0; + } + +-static int check_stack_read(struct verifier_state *state, int off, int size, +- int value_regno) ++static int check_stack_read(struct bpf_verifier_env *env, ++ struct bpf_func_state *reg_state /* func where register points to */, ++ int off, int size, int value_regno) + { +- u8 *slot_type; +- int i; +- +- slot_type = &state->stack_slot_type[MAX_BPF_STACK + off]; ++ struct bpf_verifier_state *vstate = env->cur_state; ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; ++ struct bpf_reg_state *reg; ++ u8 *stype; ++ ++ if (reg_state->allocated_stack <= slot) { ++ verbose(env, "invalid read from stack off %d+0 size %d\n", ++ off, size); ++ return -EACCES; ++ } ++ stype = reg_state->stack[spi].slot_type; ++ reg = ®_state->stack[spi].spilled_ptr; + +- if (slot_type[0] == STACK_SPILL) { ++ if (stype[0] == STACK_SPILL) { + if (size != BPF_REG_SIZE) { +- verbose("invalid size of register spill\n"); +- return -EACCES; ++ if (reg->type != SCALAR_VALUE) { ++ verbose_linfo(env, env->insn_idx, "; "); ++ verbose(env, "invalid size of register fill\n"); ++ return -EACCES; ++ } ++ if (value_regno >= 0) { ++ mark_reg_unknown(env, state->regs, value_regno); ++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; ++ } ++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); ++ return 0; + } + for (i = 1; i < BPF_REG_SIZE; i++) { +- if (slot_type[i] != STACK_SPILL) { +- verbose("corrupted spill memory\n"); ++ if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { ++ verbose(env, "corrupted spill memory\n"); + return -EACCES; + } + } + +- if (value_regno >= 0) ++ if (value_regno >= 0) { + /* restore register state from stack */ +- state->regs[value_regno] = +- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE]; +- return 0; ++ state->regs[value_regno] = *reg; ++ /* mark reg as written since spilled pointer state likely ++ * has its liveness marks cleared by is_state_visited() ++ * which resets stack/reg liveness for state transitions ++ */ ++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; ++ } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { ++ /* If value_regno==-1, the caller is asking us whether ++ * it is acceptable to use this value as a SCALAR_VALUE ++ * (e.g. for XADD). ++ * We must not allow unprivileged callers to do that ++ * with spilled pointers. ++ */ ++ verbose(env, "leaking pointer from stack off %d\n", ++ off); ++ return -EACCES; ++ } ++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); + } else { ++ int zeros = 0; ++ + for (i = 0; i < size; i++) { +- if (slot_type[i] != STACK_MISC) { +- verbose("invalid read from stack off %d+%d size %d\n", +- off, i, size); +- return -EACCES; ++ if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) ++ continue; ++ if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { ++ zeros++; ++ continue; + } ++ verbose(env, "invalid read from stack off %d+%d size %d\n", ++ off, i, size); ++ return -EACCES; ++ } ++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); ++ if (value_regno >= 0) { ++ if (zeros == size) { ++ /* any size read into register is zero extended, ++ * so the whole register == const_zero ++ */ ++ __mark_reg_const_zero(&state->regs[value_regno]); ++ /* backtracking doesn't support STACK_ZERO yet, ++ * so mark it precise here, so that later ++ * backtracking can stop here. ++ * Backtracking may not need this if this register ++ * doesn't participate in pointer adjustment. ++ * Forward propagation of precise flag is not ++ * necessary either. This mark is only to stop ++ * backtracking. Any register that contributed ++ * to const 0 was marked precise before spill. ++ */ ++ state->regs[value_regno].precise = true; ++ } else { ++ /* have read misc data from the stack */ ++ mark_reg_unknown(env, state->regs, value_regno); ++ } ++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; + } +- if (value_regno >= 0) +- /* have read misc data from the stack */ +- mark_reg_unknown_value(state->regs, value_regno); +- return 0; + } ++ return 0; ++} ++ ++static int check_stack_access(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, ++ int off, int size) ++{ ++ /* Stack accesses must be at a fixed offset, so that we ++ * can determine what type of data were returned. See ++ * check_stack_read(). ++ */ ++ if (!tnum_is_const(reg->var_off)) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "variable stack access var_off=%s off=%d size=%d\n", ++ tn_buf, off, size); ++ return -EACCES; ++ } ++ ++ if (off >= 0 || off < -MAX_BPF_STACK) { ++ verbose(env, "invalid stack off=%d size=%d\n", off, size); ++ return -EACCES; ++ } ++ ++ return 0; ++} ++ ++static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, ++ int off, int size, enum bpf_access_type type) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_map *map = regs[regno].map_ptr; ++ u32 cap = bpf_map_flags_to_cap(map); ++ ++ if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { ++ verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", ++ map->value_size, off, size); ++ return -EACCES; ++ } ++ ++ if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { ++ verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", ++ map->value_size, off, size); ++ return -EACCES; ++ } ++ ++ return 0; + } + + /* check read/write into map element returned by bpf_map_lookup_elem() */ +-static int check_map_access(struct verifier_env *env, u32 regno, int off, +- int size) ++static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off, ++ int size, bool zero_size_allowed) + { +- struct bpf_map *map = env->cur_state.regs[regno].map_ptr; ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_map *map = regs[regno].map_ptr; + +- if (off < 0 || off + size > map->value_size) { +- verbose("invalid access to map value, value_size=%d off=%d size=%d\n", ++ if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || ++ off + size > map->value_size) { ++ verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", + map->value_size, off, size); + return -EACCES; + } + return 0; + } + +-/* check access to 'struct bpf_context' fields */ +-static int check_ctx_access(struct verifier_env *env, int off, int size, +- enum bpf_access_type t) ++/* check read/write into a map element with possible variable offset */ ++static int check_map_access(struct bpf_verifier_env *env, u32 regno, ++ int off, int size, bool zero_size_allowed) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ struct bpf_reg_state *reg = &state->regs[regno]; ++ int err; ++ ++ /* We may have adjusted the register to this map value, so we ++ * need to try adding each of min_value and max_value to off ++ * to make sure our theoretical access will be safe. ++ */ ++ if (env->log.level & BPF_LOG_LEVEL) ++ print_verifier_state(env, state); ++ ++ /* The minimum value is only important with signed ++ * comparisons where we can't assume the floor of a ++ * value is 0. If we are using signed variables for our ++ * index'es we need to make sure that whatever we use ++ * will have a set floor within our range. ++ */ ++ if (reg->smin_value < 0 && ++ (reg->smin_value == S64_MIN || ++ (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || ++ reg->smin_value + off < 0)) { ++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", ++ regno); ++ return -EACCES; ++ } ++ err = __check_map_access(env, regno, reg->smin_value + off, size, ++ zero_size_allowed); ++ if (err) { ++ verbose(env, "R%d min value is outside of the array range\n", ++ regno); ++ return err; ++ } ++ ++ /* If we haven't set a max value then we need to bail since we can't be ++ * sure we won't do bad things. ++ * If reg->umax_value + off could overflow, treat that as unbounded too. ++ */ ++ if (reg->umax_value >= BPF_MAX_VAR_OFF) { ++ verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n", ++ regno); ++ return -EACCES; ++ } ++ err = __check_map_access(env, regno, reg->umax_value + off, size, ++ zero_size_allowed); ++ if (err) ++ verbose(env, "R%d max value is outside of the array range\n", ++ regno); ++ ++ if (map_value_has_spin_lock(reg->map_ptr)) { ++ u32 lock = reg->map_ptr->spin_lock_off; ++ ++ /* if any part of struct bpf_spin_lock can be touched by ++ * load/store reject this program. ++ * To check that [x1, x2) overlaps with [y1, y2) ++ * it is sufficient to check x1 < y2 && y1 < x2. ++ */ ++ if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && ++ lock < reg->umax_value + off + size) { ++ verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); ++ return -EACCES; ++ } ++ } ++ return err; ++} ++ ++#define MAX_PACKET_OFF 0xffff ++ ++static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, ++ const struct bpf_call_arg_meta *meta, ++ enum bpf_access_type t) ++{ ++ switch (env->prog->type) { ++ /* Program types only with direct read access go here! */ ++ case BPF_PROG_TYPE_LWT_IN: ++ case BPF_PROG_TYPE_LWT_OUT: ++ case BPF_PROG_TYPE_LWT_SEG6LOCAL: ++ case BPF_PROG_TYPE_SK_REUSEPORT: ++ case BPF_PROG_TYPE_FLOW_DISSECTOR: ++ case BPF_PROG_TYPE_CGROUP_SKB: ++ if (t == BPF_WRITE) ++ return false; ++ /* fallthrough */ ++ ++ /* Program types with direct read + write access go here! */ ++ case BPF_PROG_TYPE_SCHED_CLS: ++ case BPF_PROG_TYPE_SCHED_ACT: ++ case BPF_PROG_TYPE_XDP: ++ case BPF_PROG_TYPE_LWT_XMIT: ++ case BPF_PROG_TYPE_SK_SKB: ++ case BPF_PROG_TYPE_SK_MSG: ++ if (meta) ++ return meta->pkt_access; ++ ++ env->seen_direct_write = true; ++ return true; ++ ++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: ++ if (t == BPF_WRITE) ++ env->seen_direct_write = true; ++ ++ return true; ++ ++ default: ++ return false; ++ } ++} ++ ++static int __check_packet_access(struct bpf_verifier_env *env, u32 regno, ++ int off, int size, bool zero_size_allowed) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_reg_state *reg = ®s[regno]; ++ ++ if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || ++ (u64)off + size > reg->range) { ++ verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", ++ off, size, regno, reg->id, reg->off, reg->range); ++ return -EACCES; ++ } ++ return 0; ++} ++ ++static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, ++ int size, bool zero_size_allowed) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_reg_state *reg = ®s[regno]; ++ int err; ++ ++ /* We may have added a variable offset to the packet pointer; but any ++ * reg->range we have comes after that. We are only checking the fixed ++ * offset. ++ */ ++ ++ /* We don't allow negative numbers, because we aren't tracking enough ++ * detail to prove they're safe. ++ */ ++ if (reg->smin_value < 0) { ++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", ++ regno); ++ return -EACCES; ++ } ++ err = __check_packet_access(env, regno, off, size, zero_size_allowed); ++ if (err) { ++ verbose(env, "R%d offset is outside of the packet\n", regno); ++ return err; ++ } ++ ++ /* __check_packet_access has made sure "off + size - 1" is within u16. ++ * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, ++ * otherwise find_good_pkt_pointers would have refused to set range info ++ * that __check_packet_access would have rejected this pkt access. ++ * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. ++ */ ++ env->prog->aux->max_pkt_offset = ++ max_t(u32, env->prog->aux->max_pkt_offset, ++ off + reg->umax_value + size - 1); ++ ++ return err; ++} ++ ++/* check access to 'struct bpf_context' fields. Supports fixed offsets only */ ++static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, ++ enum bpf_access_type t, enum bpf_reg_type *reg_type) + { +- if (env->prog->aux->ops->is_valid_access && +- env->prog->aux->ops->is_valid_access(off, size, t)) ++ struct bpf_insn_access_aux info = { ++ .reg_type = *reg_type, ++ }; ++ ++ if (env->ops->is_valid_access && ++ env->ops->is_valid_access(off, size, t, env->prog, &info)) { ++ /* A non zero info.ctx_field_size indicates that this field is a ++ * candidate for later verifier transformation to load the whole ++ * field and then apply a mask when accessed with a narrower ++ * access than actual ctx access size. A zero info.ctx_field_size ++ * will only allow for whole field access and rejects any other ++ * type of narrower access. ++ */ ++ *reg_type = info.reg_type; ++ ++ env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; ++ /* remember the offset of last byte accessed in ctx */ ++ if (env->prog->aux->max_ctx_offset < off + size) ++ env->prog->aux->max_ctx_offset = off + size; + return 0; ++ } + +- verbose("invalid bpf_context access off=%d size=%d\n", off, size); ++ verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); + return -EACCES; + } + +-static bool is_pointer_value(struct verifier_env *env, int regno) ++static int check_flow_keys_access(struct bpf_verifier_env *env, int off, ++ int size) + { +- if (env->allow_ptr_leaks) +- return false; ++ if (size < 0 || off < 0 || ++ (u64)off + size > sizeof(struct bpf_flow_keys)) { ++ verbose(env, "invalid access to flow keys off=%d size=%d\n", ++ off, size); ++ return -EACCES; ++ } ++ return 0; ++} + +- switch (env->cur_state.regs[regno].type) { +- case UNKNOWN_VALUE: +- case CONST_IMM: +- return false; ++static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, ++ u32 regno, int off, int size, ++ enum bpf_access_type t) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_reg_state *reg = ®s[regno]; ++ struct bpf_insn_access_aux info = {}; ++ bool valid; ++ ++ if (reg->smin_value < 0) { ++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", ++ regno); ++ return -EACCES; ++ } ++ ++ switch (reg->type) { ++ case PTR_TO_SOCK_COMMON: ++ valid = bpf_sock_common_is_valid_access(off, size, t, &info); ++ break; + default: +- return true; ++ valid = false; ++ } ++ ++ ++ if (valid) { ++ env->insn_aux_data[insn_idx].ctx_field_size = ++ info.ctx_field_size; ++ return 0; ++ } ++ ++ verbose(env, "R%d invalid %s access off=%d size=%d\n", ++ regno, reg_type_str[reg->type], off, size); ++ ++ return -EACCES; ++} ++ ++static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) ++{ ++ return cur_regs(env) + regno; ++} ++ ++static bool is_pointer_value(struct bpf_verifier_env *env, int regno) ++{ ++ return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); ++} ++ ++static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) ++{ ++ const struct bpf_reg_state *reg = reg_state(env, regno); ++ ++ return reg->type == PTR_TO_CTX; ++} ++ ++static bool is_sk_reg(struct bpf_verifier_env *env, int regno) ++{ ++ const struct bpf_reg_state *reg = reg_state(env, regno); ++ ++ return type_is_sk_pointer(reg->type); ++} ++ ++static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) ++{ ++ const struct bpf_reg_state *reg = reg_state(env, regno); ++ ++ return type_is_pkt_pointer(reg->type); ++} ++ ++static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) ++{ ++ const struct bpf_reg_state *reg = reg_state(env, regno); ++ ++ /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ ++ return reg->type == PTR_TO_FLOW_KEYS; ++} ++ ++static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, ++ int off, int size, bool strict) ++{ ++ struct tnum reg_off; ++ int ip_align; ++ ++ /* Byte size accesses are always allowed. */ ++ if (!strict || size == 1) ++ return 0; ++ ++ /* For platforms that do not have a Kconfig enabling ++ * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of ++ * NET_IP_ALIGN is universally set to '2'. And on platforms ++ * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get ++ * to this code only in strict mode where we want to emulate ++ * the NET_IP_ALIGN==2 checking. Therefore use an ++ * unconditional IP align value of '2'. ++ */ ++ ip_align = 2; ++ ++ reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); ++ if (!tnum_is_aligned(reg_off, size)) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, ++ "misaligned packet access off %d+%s+%d+%d size %d\n", ++ ip_align, tn_buf, reg->off, off, size); ++ return -EACCES; ++ } ++ ++ return 0; ++} ++ ++static int check_generic_ptr_alignment(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, ++ const char *pointer_desc, ++ int off, int size, bool strict) ++{ ++ struct tnum reg_off; ++ ++ /* Byte size accesses are always allowed. */ ++ if (!strict || size == 1) ++ return 0; ++ ++ reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); ++ if (!tnum_is_aligned(reg_off, size)) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", ++ pointer_desc, tn_buf, reg->off, off, size); ++ return -EACCES; ++ } ++ ++ return 0; ++} ++ ++static int check_ptr_alignment(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, int off, ++ int size, bool strict_alignment_once) ++{ ++ bool strict = env->strict_alignment || strict_alignment_once; ++ const char *pointer_desc = ""; ++ ++ switch (reg->type) { ++ case PTR_TO_PACKET: ++ case PTR_TO_PACKET_META: ++ /* Special case, because of NET_IP_ALIGN. Given metadata sits ++ * right in front, treat it the very same way. ++ */ ++ return check_pkt_ptr_alignment(env, reg, off, size, strict); ++ case PTR_TO_FLOW_KEYS: ++ pointer_desc = "flow keys "; ++ break; ++ case PTR_TO_MAP_VALUE: ++ pointer_desc = "value "; ++ break; ++ case PTR_TO_CTX: ++ pointer_desc = "context "; ++ break; ++ case PTR_TO_STACK: ++ pointer_desc = "stack "; ++ /* The stack spill tracking logic in check_stack_write() ++ * and check_stack_read() relies on stack accesses being ++ * aligned. ++ */ ++ strict = true; ++ break; ++ case PTR_TO_SOCKET: ++ pointer_desc = "sock "; ++ break; ++ case PTR_TO_SOCK_COMMON: ++ pointer_desc = "sock_common "; ++ break; ++ case PTR_TO_TCP_SOCK: ++ pointer_desc = "tcp_sock "; ++ break; ++ case PTR_TO_XDP_SOCK: ++ pointer_desc = "xdp_sock "; ++ break; ++ default: ++ break; ++ } ++ return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, ++ strict); ++} ++ ++static int update_stack_depth(struct bpf_verifier_env *env, ++ const struct bpf_func_state *func, ++ int off) ++{ ++ u16 stack = env->subprog_info[func->subprogno].stack_depth; ++ ++ if (stack >= -off) ++ return 0; ++ ++ /* update known max for given subprogram */ ++ env->subprog_info[func->subprogno].stack_depth = -off; ++ return 0; ++} ++ ++/* starting from main bpf function walk all instructions of the function ++ * and recursively walk all callees that given function can call. ++ * Ignore jump and exit insns. ++ * Since recursion is prevented by check_cfg() this algorithm ++ * only needs a local stack of MAX_CALL_FRAMES to remember callsites ++ */ ++static int check_max_stack_depth(struct bpf_verifier_env *env) ++{ ++ int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; ++ struct bpf_subprog_info *subprog = env->subprog_info; ++ struct bpf_insn *insn = env->prog->insnsi; ++ int ret_insn[MAX_CALL_FRAMES]; ++ int ret_prog[MAX_CALL_FRAMES]; ++ ++process_func: ++ /* protect against potential stack overflow that might happen when ++ * bpf2bpf calls get combined with tailcalls. Limit the caller's stack ++ * depth for such case down to 256 so that the worst case scenario ++ * would result in 8k stack size (32 which is tailcall limit * 256 = ++ * 8k). ++ * ++ * To get the idea what might happen, see an example: ++ * func1 -> sub rsp, 128 ++ * subfunc1 -> sub rsp, 256 ++ * tailcall1 -> add rsp, 256 ++ * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) ++ * subfunc2 -> sub rsp, 64 ++ * subfunc22 -> sub rsp, 128 ++ * tailcall2 -> add rsp, 128 ++ * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) ++ * ++ * tailcall will unwind the current stack frame but it will not get rid ++ * of caller's stack as shown on the example above. ++ */ ++ if (idx && subprog[idx].has_tail_call && depth >= 256) { ++ verbose(env, ++ "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", ++ depth); ++ return -EACCES; ++ } ++ /* round up to 32-bytes, since this is granularity ++ * of interpreter stack size ++ */ ++ depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); ++ if (depth > MAX_BPF_STACK) { ++ verbose(env, "combined stack size of %d calls is %d. Too large\n", ++ frame + 1, depth); ++ return -EACCES; ++ } ++continue_func: ++ subprog_end = subprog[idx + 1].start; ++ for (; i < subprog_end; i++) { ++ if (insn[i].code != (BPF_JMP | BPF_CALL)) ++ continue; ++ if (insn[i].src_reg != BPF_PSEUDO_CALL) ++ continue; ++ /* remember insn and function to return to */ ++ ret_insn[frame] = i + 1; ++ ret_prog[frame] = idx; ++ ++ /* find the callee */ ++ i = i + insn[i].imm + 1; ++ idx = find_subprog(env, i); ++ if (idx < 0) { ++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", ++ i); ++ return -EFAULT; ++ } ++ frame++; ++ if (frame >= MAX_CALL_FRAMES) { ++ verbose(env, "the call stack of %d frames is too deep !\n", ++ frame); ++ return -E2BIG; ++ } ++ goto process_func; ++ } ++ /* end of for() loop means the last insn of the 'subprog' ++ * was reached. Doesn't matter whether it was JA or EXIT ++ */ ++ if (frame == 0) ++ return 0; ++ depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); ++ frame--; ++ i = ret_insn[frame]; ++ idx = ret_prog[frame]; ++ goto continue_func; ++} ++ ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++static int get_callee_stack_depth(struct bpf_verifier_env *env, ++ const struct bpf_insn *insn, int idx) ++{ ++ int start = idx + insn->imm + 1, subprog; ++ ++ subprog = find_subprog(env, start); ++ if (subprog < 0) { ++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", ++ start); ++ return -EFAULT; + } ++ return env->subprog_info[subprog].stack_depth; ++} ++#endif ++ ++static int check_ctx_reg(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, int regno) ++{ ++ /* Access to ctx or passing it to a helper is only allowed in ++ * its original, unmodified form. ++ */ ++ ++ if (reg->off) { ++ verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", ++ regno, reg->off); ++ return -EACCES; ++ } ++ ++ if (!tnum_is_const(reg->var_off) || reg->var_off.value) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); ++ return -EACCES; ++ } ++ ++ return 0; ++} ++ ++static int check_tp_buffer_access(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, ++ int regno, int off, int size) ++{ ++ if (off < 0) { ++ verbose(env, ++ "R%d invalid tracepoint buffer access: off=%d, size=%d", ++ regno, off, size); ++ return -EACCES; ++ } ++ if (!tnum_is_const(reg->var_off) || reg->var_off.value) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, ++ "R%d invalid variable buffer offset: off=%d, var_off=%s", ++ regno, off, tn_buf); ++ return -EACCES; ++ } ++ if (off + size > env->prog->aux->max_tp_access) ++ env->prog->aux->max_tp_access = off + size; ++ ++ return 0; ++} ++ ++ ++/* truncate register to smaller size (in bytes) ++ * must be called with size < BPF_REG_SIZE ++ */ ++static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) ++{ ++ u64 mask; ++ ++ /* clear high bits in bit representation */ ++ reg->var_off = tnum_cast(reg->var_off, size); ++ ++ /* fix arithmetic bounds */ ++ mask = ((u64)1 << (size * 8)) - 1; ++ if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { ++ reg->umin_value &= mask; ++ reg->umax_value &= mask; ++ } else { ++ reg->umin_value = 0; ++ reg->umax_value = mask; ++ } ++ reg->smin_value = reg->umin_value; ++ reg->smax_value = reg->umax_value; + } + + /* check whether memory at (regno + off) is accessible for t = (read | write) +@@ -680,225 +2776,692 @@ static bool is_pointer_value(struct veri + * if t==write && value_regno==-1, some unknown value is stored into memory + * if t==read && value_regno==-1, don't care what we read from memory + */ +-static int check_mem_access(struct verifier_env *env, u32 regno, int off, +- int bpf_size, enum bpf_access_type t, +- int value_regno) ++static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, ++ int off, int bpf_size, enum bpf_access_type t, ++ int value_regno, bool strict_alignment_once) + { +- struct verifier_state *state = &env->cur_state; ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_reg_state *reg = regs + regno; ++ struct bpf_func_state *state; + int size, err = 0; + +- if (state->regs[regno].type == PTR_TO_STACK) +- off += state->regs[regno].imm; +- + size = bpf_size_to_bytes(bpf_size); + if (size < 0) + return size; + +- if (off % size != 0) { +- verbose("misaligned access off %d size %d\n", off, size); +- return -EACCES; +- } ++ /* alignment checks will add in reg->off themselves */ ++ err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); ++ if (err) ++ return err; ++ ++ /* for access checks, reg->off is just part of off */ ++ off += reg->off; + +- if (state->regs[regno].type == PTR_TO_MAP_VALUE) { ++ if (reg->type == PTR_TO_MAP_VALUE) { + if (t == BPF_WRITE && value_regno >= 0 && + is_pointer_value(env, value_regno)) { +- verbose("R%d leaks addr into map\n", value_regno); ++ verbose(env, "R%d leaks addr into map\n", value_regno); + return -EACCES; + } +- err = check_map_access(env, regno, off, size); ++ err = check_map_access_type(env, regno, off, size, t); ++ if (err) ++ return err; ++ err = check_map_access(env, regno, off, size, false); + if (!err && t == BPF_READ && value_regno >= 0) +- mark_reg_unknown_value(state->regs, value_regno); ++ mark_reg_unknown(env, regs, value_regno); ++ ++ } else if (reg->type == PTR_TO_CTX) { ++ enum bpf_reg_type reg_type = SCALAR_VALUE; + +- } else if (state->regs[regno].type == PTR_TO_CTX) { + if (t == BPF_WRITE && value_regno >= 0 && + is_pointer_value(env, value_regno)) { +- verbose("R%d leaks addr into ctx\n", value_regno); ++ verbose(env, "R%d leaks addr into ctx\n", value_regno); + return -EACCES; + } +- err = check_ctx_access(env, off, size, t); +- if (!err && t == BPF_READ && value_regno >= 0) +- mark_reg_unknown_value(state->regs, value_regno); + +- } else if (state->regs[regno].type == FRAME_PTR || +- state->regs[regno].type == PTR_TO_STACK) { +- if (off >= 0 || off < -MAX_BPF_STACK) { +- verbose("invalid stack off=%d size=%d\n", off, size); ++ err = check_ctx_reg(env, reg, regno); ++ if (err < 0) ++ return err; ++ ++ err = check_ctx_access(env, insn_idx, off, size, t, ®_type); ++ if (!err && t == BPF_READ && value_regno >= 0) { ++ /* ctx access returns either a scalar, or a ++ * PTR_TO_PACKET[_META,_END]. In the latter ++ * case, we know the offset is zero. ++ */ ++ if (reg_type == SCALAR_VALUE) { ++ mark_reg_unknown(env, regs, value_regno); ++ } else { ++ mark_reg_known_zero(env, regs, ++ value_regno); ++ if (reg_type_may_be_null(reg_type)) ++ regs[value_regno].id = ++env->id_gen; ++ /* A load of ctx field could have different ++ * actual load size with the one encoded in the ++ * insn. When the dst is PTR, it is for sure not ++ * a sub-register. ++ */ ++ regs[value_regno].subreg_def = DEF_NOT_SUBREG; ++ } ++ regs[value_regno].type = reg_type; ++ } ++ ++ } else if (reg->type == PTR_TO_STACK) { ++ off += reg->var_off.value; ++ err = check_stack_access(env, reg, off, size); ++ if (err) ++ return err; ++ ++ state = func(env, reg); ++ err = update_stack_depth(env, state, off); ++ if (err) ++ return err; ++ ++ if (t == BPF_WRITE) ++ err = check_stack_write(env, state, off, size, ++ value_regno, insn_idx); ++ else ++ err = check_stack_read(env, state, off, size, ++ value_regno); ++ } else if (reg_is_pkt_pointer(reg)) { ++ if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { ++ verbose(env, "cannot write into packet\n"); ++ return -EACCES; ++ } ++ if (t == BPF_WRITE && value_regno >= 0 && ++ is_pointer_value(env, value_regno)) { ++ verbose(env, "R%d leaks addr into packet\n", ++ value_regno); ++ return -EACCES; ++ } ++ err = check_packet_access(env, regno, off, size, false); ++ if (!err && t == BPF_READ && value_regno >= 0) ++ mark_reg_unknown(env, regs, value_regno); ++ } else if (reg->type == PTR_TO_FLOW_KEYS) { ++ if (t == BPF_WRITE && value_regno >= 0 && ++ is_pointer_value(env, value_regno)) { ++ verbose(env, "R%d leaks addr into flow keys\n", ++ value_regno); + return -EACCES; + } ++ ++ err = check_flow_keys_access(env, off, size); ++ if (!err && t == BPF_READ && value_regno >= 0) ++ mark_reg_unknown(env, regs, value_regno); ++ } else if (type_is_sk_pointer(reg->type)) { + if (t == BPF_WRITE) { +- if (!env->allow_ptr_leaks && +- state->stack_slot_type[MAX_BPF_STACK + off] == STACK_SPILL && +- size != BPF_REG_SIZE) { +- verbose("attempt to corrupt spilled pointer on stack\n"); +- return -EACCES; +- } +- err = check_stack_write(state, off, size, value_regno); +- } else { +- err = check_stack_read(state, off, size, value_regno); ++ verbose(env, "R%d cannot write into %s\n", ++ regno, reg_type_str[reg->type]); ++ return -EACCES; + } ++ err = check_sock_access(env, insn_idx, regno, off, size, t); ++ if (!err && value_regno >= 0) ++ mark_reg_unknown(env, regs, value_regno); ++ } else if (reg->type == PTR_TO_TP_BUFFER) { ++ err = check_tp_buffer_access(env, reg, regno, off, size); ++ if (!err && t == BPF_READ && value_regno >= 0) ++ mark_reg_unknown(env, regs, value_regno); + } else { +- verbose("R%d invalid mem access '%s'\n", +- regno, reg_type_str[state->regs[regno].type]); ++ verbose(env, "R%d invalid mem access '%s'\n", regno, ++ reg_type_str[reg->type]); + return -EACCES; + } ++ ++ if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && ++ regs[value_regno].type == SCALAR_VALUE) { ++ /* b/h/w load zero-extends, mark upper bits as known 0 */ ++ coerce_reg_to_size(®s[value_regno], size); ++ } + return err; + } + +-static int check_xadd(struct verifier_env *env, struct bpf_insn *insn) ++static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) + { +- struct reg_state *regs = env->cur_state.regs; + int err; + + if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || + insn->imm != 0) { +- verbose("BPF_XADD uses reserved fields\n"); ++ verbose(env, "BPF_XADD uses reserved fields\n"); + return -EINVAL; + } + + /* check src1 operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + + /* check src2 operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + ++ if (is_pointer_value(env, insn->src_reg)) { ++ verbose(env, "R%d leaks addr into mem\n", insn->src_reg); ++ return -EACCES; ++ } ++ ++ if (is_ctx_reg(env, insn->dst_reg) || ++ is_pkt_reg(env, insn->dst_reg) || ++ is_flow_key_reg(env, insn->dst_reg) || ++ is_sk_reg(env, insn->dst_reg)) { ++ verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", ++ insn->dst_reg, ++ reg_type_str[reg_state(env, insn->dst_reg)->type]); ++ return -EACCES; ++ } ++ + /* check whether atomic_add can read the memory */ +- err = check_mem_access(env, insn->dst_reg, insn->off, +- BPF_SIZE(insn->code), BPF_READ, -1); ++ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, ++ BPF_SIZE(insn->code), BPF_READ, -1, true); + if (err) + return err; + + /* check whether atomic_add can write into the same memory */ +- return check_mem_access(env, insn->dst_reg, insn->off, +- BPF_SIZE(insn->code), BPF_WRITE, -1); ++ return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, ++ BPF_SIZE(insn->code), BPF_WRITE, -1, true); ++} ++ ++static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, ++ int off, int access_size, ++ bool zero_size_allowed) ++{ ++ struct bpf_reg_state *reg = reg_state(env, regno); ++ ++ if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || ++ access_size < 0 || (access_size == 0 && !zero_size_allowed)) { ++ if (tnum_is_const(reg->var_off)) { ++ verbose(env, "invalid stack type R%d off=%d access_size=%d\n", ++ regno, off, access_size); ++ } else { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", ++ regno, tn_buf, access_size); ++ } ++ return -EACCES; ++ } ++ return 0; + } + + /* when register 'regno' is passed into function that will read 'access_size' + * bytes from that pointer, make sure that it's within stack boundary +- * and all elements of stack are initialized ++ * and all elements of stack are initialized. ++ * Unlike most pointer bounds-checking functions, this one doesn't take an ++ * 'off' argument, so it has to add in reg->off itself. + */ +-static int check_stack_boundary(struct verifier_env *env, +- int regno, int access_size) ++static int check_stack_boundary(struct bpf_verifier_env *env, int regno, ++ int access_size, bool zero_size_allowed, ++ struct bpf_call_arg_meta *meta) + { +- struct verifier_state *state = &env->cur_state; +- struct reg_state *regs = state->regs; +- int off, i; ++ struct bpf_reg_state *reg = reg_state(env, regno); ++ struct bpf_func_state *state = func(env, reg); ++ int err, min_off, max_off, i, j, slot, spi; ++ ++ if (reg->type != PTR_TO_STACK) { ++ /* Allow zero-byte read from NULL, regardless of pointer type */ ++ if (zero_size_allowed && access_size == 0 && ++ register_is_null(reg)) ++ return 0; + +- if (regs[regno].type != PTR_TO_STACK) ++ verbose(env, "R%d type=%s expected=%s\n", regno, ++ reg_type_str[reg->type], ++ reg_type_str[PTR_TO_STACK]); + return -EACCES; ++ } + +- off = regs[regno].imm; +- if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || +- access_size <= 0) { +- verbose("invalid stack type R%d off=%d access_size=%d\n", +- regno, off, access_size); ++ if (tnum_is_const(reg->var_off)) { ++ min_off = max_off = reg->var_off.value + reg->off; ++ err = __check_stack_boundary(env, regno, min_off, access_size, ++ zero_size_allowed); ++ if (err) ++ return err; ++ } else { ++ /* Variable offset is prohibited for unprivileged mode for ++ * simplicity since it requires corresponding support in ++ * Spectre masking for stack ALU. ++ * See also retrieve_ptr_limit(). ++ */ ++ if (!env->allow_ptr_leaks) { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", ++ regno, tn_buf); ++ return -EACCES; ++ } ++ /* Only initialized buffer on stack is allowed to be accessed ++ * with variable offset. With uninitialized buffer it's hard to ++ * guarantee that whole memory is marked as initialized on ++ * helper return since specific bounds are unknown what may ++ * cause uninitialized stack leaking. ++ */ ++ if (meta && meta->raw_mode) ++ meta = NULL; ++ ++ if (reg->smax_value >= BPF_MAX_VAR_OFF || ++ reg->smax_value <= -BPF_MAX_VAR_OFF) { ++ verbose(env, "R%d unbounded indirect variable offset stack access\n", ++ regno); ++ return -EACCES; ++ } ++ min_off = reg->smin_value + reg->off; ++ max_off = reg->smax_value + reg->off; ++ err = __check_stack_boundary(env, regno, min_off, access_size, ++ zero_size_allowed); ++ if (err) { ++ verbose(env, "R%d min value is outside of stack bound\n", ++ regno); ++ return err; ++ } ++ err = __check_stack_boundary(env, regno, max_off, access_size, ++ zero_size_allowed); ++ if (err) { ++ verbose(env, "R%d max value is outside of stack bound\n", ++ regno); ++ return err; ++ } ++ } ++ ++ if (meta && meta->raw_mode) { ++ meta->access_size = access_size; ++ meta->regno = regno; ++ return 0; ++ } ++ ++ for (i = min_off; i < max_off + access_size; i++) { ++ u8 *stype; ++ ++ slot = -i - 1; ++ spi = slot / BPF_REG_SIZE; ++ if (state->allocated_stack <= slot) ++ goto err; ++ stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; ++ if (*stype == STACK_MISC) ++ goto mark; ++ if (*stype == STACK_ZERO) { ++ /* helper can write anything into the stack */ ++ *stype = STACK_MISC; ++ goto mark; ++ } ++ if (state->stack[spi].slot_type[0] == STACK_SPILL && ++ state->stack[spi].spilled_ptr.type == SCALAR_VALUE) { ++ __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); ++ for (j = 0; j < BPF_REG_SIZE; j++) ++ state->stack[spi].slot_type[j] = STACK_MISC; ++ goto mark; ++ } ++ ++err: ++ if (tnum_is_const(reg->var_off)) { ++ verbose(env, "invalid indirect read from stack off %d+%d size %d\n", ++ min_off, i - min_off, access_size); ++ } else { ++ char tn_buf[48]; ++ ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", ++ tn_buf, i - min_off, access_size); ++ } + return -EACCES; ++mark: ++ /* reading any byte out of 8-byte 'spill_slot' will cause ++ * the whole slot to be marked as 'read' ++ */ ++ mark_reg_read(env, &state->stack[spi].spilled_ptr, ++ state->stack[spi].spilled_ptr.parent, ++ REG_LIVE_READ64); + } ++ return update_stack_depth(env, state, min_off); ++} ++ ++static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, ++ int access_size, bool zero_size_allowed, ++ struct bpf_call_arg_meta *meta) ++{ ++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; + +- for (i = 0; i < access_size; i++) { +- if (state->stack_slot_type[MAX_BPF_STACK + off + i] != STACK_MISC) { +- verbose("invalid indirect read from stack off %d+%d size %d\n", +- off, i, access_size); ++ switch (reg->type) { ++ case PTR_TO_PACKET: ++ case PTR_TO_PACKET_META: ++ return check_packet_access(env, regno, reg->off, access_size, ++ zero_size_allowed); ++ case PTR_TO_MAP_VALUE: ++ if (check_map_access_type(env, regno, reg->off, access_size, ++ meta && meta->raw_mode ? BPF_WRITE : ++ BPF_READ)) + return -EACCES; ++ return check_map_access(env, regno, reg->off, access_size, ++ zero_size_allowed); ++ default: /* scalar_value|ptr_to_stack or invalid ptr */ ++ return check_stack_boundary(env, regno, access_size, ++ zero_size_allowed, meta); ++ } ++} ++ ++/* Implementation details: ++ * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL ++ * Two bpf_map_lookups (even with the same key) will have different reg->id. ++ * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after ++ * value_or_null->value transition, since the verifier only cares about ++ * the range of access to valid map value pointer and doesn't care about actual ++ * address of the map element. ++ * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps ++ * reg->id > 0 after value_or_null->value transition. By doing so ++ * two bpf_map_lookups will be considered two different pointers that ++ * point to different bpf_spin_locks. ++ * The verifier allows taking only one bpf_spin_lock at a time to avoid ++ * dead-locks. ++ * Since only one bpf_spin_lock is allowed the checks are simpler than ++ * reg_is_refcounted() logic. The verifier needs to remember only ++ * one spin_lock instead of array of acquired_refs. ++ * cur_state->active_spin_lock remembers which map value element got locked ++ * and clears it after bpf_spin_unlock. ++ */ ++static int process_spin_lock(struct bpf_verifier_env *env, int regno, ++ bool is_lock) ++{ ++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; ++ struct bpf_verifier_state *cur = env->cur_state; ++ bool is_const = tnum_is_const(reg->var_off); ++ struct bpf_map *map = reg->map_ptr; ++ u64 val = reg->var_off.value; ++ ++ if (reg->type != PTR_TO_MAP_VALUE) { ++ verbose(env, "R%d is not a pointer to map_value\n", regno); ++ return -EINVAL; ++ } ++ if (!is_const) { ++ verbose(env, ++ "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", ++ regno); ++ return -EINVAL; ++ } ++ if (!map->btf) { ++ verbose(env, ++ "map '%s' has to have BTF in order to use bpf_spin_lock\n", ++ map->name); ++ return -EINVAL; ++ } ++ if (!map_value_has_spin_lock(map)) { ++ if (map->spin_lock_off == -E2BIG) ++ verbose(env, ++ "map '%s' has more than one 'struct bpf_spin_lock'\n", ++ map->name); ++ else if (map->spin_lock_off == -ENOENT) ++ verbose(env, ++ "map '%s' doesn't have 'struct bpf_spin_lock'\n", ++ map->name); ++ else ++ verbose(env, ++ "map '%s' is not a struct type or bpf_spin_lock is mangled\n", ++ map->name); ++ return -EINVAL; ++ } ++ if (map->spin_lock_off != val + reg->off) { ++ verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", ++ val + reg->off); ++ return -EINVAL; ++ } ++ if (is_lock) { ++ if (cur->active_spin_lock) { ++ verbose(env, ++ "Locking two bpf_spin_locks are not allowed\n"); ++ return -EINVAL; ++ } ++ cur->active_spin_lock = reg->id; ++ } else { ++ if (!cur->active_spin_lock) { ++ verbose(env, "bpf_spin_unlock without taking a lock\n"); ++ return -EINVAL; + } ++ if (cur->active_spin_lock != reg->id) { ++ verbose(env, "bpf_spin_unlock of different lock\n"); ++ return -EINVAL; ++ } ++ cur->active_spin_lock = 0; + } + return 0; + } + +-static int check_func_arg(struct verifier_env *env, u32 regno, +- enum bpf_arg_type arg_type, struct bpf_map **mapp) ++static bool arg_type_is_mem_ptr(enum bpf_arg_type type) ++{ ++ return type == ARG_PTR_TO_MEM || ++ type == ARG_PTR_TO_MEM_OR_NULL || ++ type == ARG_PTR_TO_UNINIT_MEM; ++} ++ ++static bool arg_type_is_mem_size(enum bpf_arg_type type) ++{ ++ return type == ARG_CONST_SIZE || ++ type == ARG_CONST_SIZE_OR_ZERO; ++} ++ ++static bool arg_type_is_int_ptr(enum bpf_arg_type type) ++{ ++ return type == ARG_PTR_TO_INT || ++ type == ARG_PTR_TO_LONG; ++} ++ ++static int int_ptr_type_to_size(enum bpf_arg_type type) ++{ ++ if (type == ARG_PTR_TO_INT) ++ return sizeof(u32); ++ else if (type == ARG_PTR_TO_LONG) ++ return sizeof(u64); ++ ++ return -EINVAL; ++} ++ ++static int check_func_arg(struct bpf_verifier_env *env, u32 regno, ++ enum bpf_arg_type arg_type, ++ struct bpf_call_arg_meta *meta) + { +- struct reg_state *reg = env->cur_state.regs + regno; +- enum bpf_reg_type expected_type; ++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; ++ enum bpf_reg_type expected_type, type = reg->type; + int err = 0; + + if (arg_type == ARG_DONTCARE) + return 0; + +- if (reg->type == NOT_INIT) { +- verbose("R%d !read_ok\n", regno); +- return -EACCES; +- } ++ err = check_reg_arg(env, regno, SRC_OP); ++ if (err) ++ return err; + + if (arg_type == ARG_ANYTHING) { + if (is_pointer_value(env, regno)) { +- verbose("R%d leaks addr into helper function\n", regno); ++ verbose(env, "R%d leaks addr into helper function\n", ++ regno); + return -EACCES; + } + return 0; + } + +- if (arg_type == ARG_PTR_TO_STACK || arg_type == ARG_PTR_TO_MAP_KEY || +- arg_type == ARG_PTR_TO_MAP_VALUE) { ++ if (type_is_pkt_pointer(type) && ++ !may_access_direct_pkt_data(env, meta, BPF_READ)) { ++ verbose(env, "helper access to the packet is not allowed\n"); ++ return -EACCES; ++ } ++ ++ if (arg_type == ARG_PTR_TO_MAP_KEY || ++ arg_type == ARG_PTR_TO_MAP_VALUE || ++ arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || ++ arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { + expected_type = PTR_TO_STACK; +- } else if (arg_type == ARG_CONST_STACK_SIZE) { +- expected_type = CONST_IMM; ++ if (register_is_null(reg) && ++ arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) ++ /* final test in check_stack_boundary() */; ++ else if (!type_is_pkt_pointer(type) && ++ type != PTR_TO_MAP_VALUE && ++ type != expected_type) ++ goto err_type; ++ } else if (arg_type == ARG_CONST_SIZE || ++ arg_type == ARG_CONST_SIZE_OR_ZERO) { ++ expected_type = SCALAR_VALUE; ++ if (type != expected_type) ++ goto err_type; + } else if (arg_type == ARG_CONST_MAP_PTR) { + expected_type = CONST_PTR_TO_MAP; ++ if (type != expected_type) ++ goto err_type; + } else if (arg_type == ARG_PTR_TO_CTX) { + expected_type = PTR_TO_CTX; ++ if (type != expected_type) ++ goto err_type; ++ err = check_ctx_reg(env, reg, regno); ++ if (err < 0) ++ return err; ++ } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) { ++ expected_type = PTR_TO_SOCK_COMMON; ++ /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */ ++ if (!type_is_sk_pointer(type)) ++ goto err_type; ++ if (reg->ref_obj_id) { ++ if (meta->ref_obj_id) { ++ verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", ++ regno, reg->ref_obj_id, ++ meta->ref_obj_id); ++ return -EFAULT; ++ } ++ meta->ref_obj_id = reg->ref_obj_id; ++ } ++ } else if (arg_type == ARG_PTR_TO_SOCKET) { ++ expected_type = PTR_TO_SOCKET; ++ if (type != expected_type) ++ goto err_type; ++ } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { ++ if (meta->func_id == BPF_FUNC_spin_lock) { ++ if (process_spin_lock(env, regno, true)) ++ return -EACCES; ++ } else if (meta->func_id == BPF_FUNC_spin_unlock) { ++ if (process_spin_lock(env, regno, false)) ++ return -EACCES; ++ } else { ++ verbose(env, "verifier internal error\n"); ++ return -EFAULT; ++ } ++ } else if (arg_type_is_mem_ptr(arg_type)) { ++ expected_type = PTR_TO_STACK; ++ /* One exception here. In case function allows for NULL to be ++ * passed in as argument, it's a SCALAR_VALUE type. Final test ++ * happens during stack boundary checking. ++ */ ++ if (register_is_null(reg) && ++ arg_type == ARG_PTR_TO_MEM_OR_NULL) ++ /* final test in check_stack_boundary() */; ++ else if (!type_is_pkt_pointer(type) && ++ type != PTR_TO_MAP_VALUE && ++ type != expected_type) ++ goto err_type; ++ meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM; ++ } else if (arg_type_is_int_ptr(arg_type)) { ++ expected_type = PTR_TO_STACK; ++ if (!type_is_pkt_pointer(type) && ++ type != PTR_TO_MAP_VALUE && ++ type != expected_type) ++ goto err_type; + } else { +- verbose("unsupported arg_type %d\n", arg_type); ++ verbose(env, "unsupported arg_type %d\n", arg_type); + return -EFAULT; + } + +- if (reg->type != expected_type) { +- verbose("R%d type=%s expected=%s\n", regno, +- reg_type_str[reg->type], reg_type_str[expected_type]); +- return -EACCES; +- } +- + if (arg_type == ARG_CONST_MAP_PTR) { + /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ +- *mapp = reg->map_ptr; +- ++ meta->map_ptr = reg->map_ptr; + } else if (arg_type == ARG_PTR_TO_MAP_KEY) { + /* bpf_map_xxx(..., map_ptr, ..., key) call: + * check that [key, key + map->key_size) are within + * stack limits and initialized + */ +- if (!*mapp) { ++ if (!meta->map_ptr) { + /* in function declaration map_ptr must come before + * map_key, so that it's verified and known before + * we have to check map_key here. Otherwise it means + * that kernel subsystem misconfigured verifier + */ +- verbose("invalid map_ptr to access map->key\n"); ++ verbose(env, "invalid map_ptr to access map->key\n"); + return -EACCES; + } +- err = check_stack_boundary(env, regno, (*mapp)->key_size); +- +- } else if (arg_type == ARG_PTR_TO_MAP_VALUE) { ++ err = check_helper_mem_access(env, regno, ++ meta->map_ptr->key_size, false, ++ NULL); ++ } else if (arg_type == ARG_PTR_TO_MAP_VALUE || ++ (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && ++ !register_is_null(reg)) || ++ arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { + /* bpf_map_xxx(..., map_ptr, ..., value) call: + * check [value, value + map->value_size) validity + */ +- if (!*mapp) { ++ if (!meta->map_ptr) { + /* kernel subsystem misconfigured verifier */ +- verbose("invalid map_ptr to access map->value\n"); ++ verbose(env, "invalid map_ptr to access map->value\n"); + return -EACCES; + } +- err = check_stack_boundary(env, regno, (*mapp)->value_size); ++ meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); ++ err = check_helper_mem_access(env, regno, ++ meta->map_ptr->value_size, false, ++ meta); ++ } else if (arg_type_is_mem_size(arg_type)) { ++ bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); + +- } else if (arg_type == ARG_CONST_STACK_SIZE) { +- /* bpf_xxx(..., buf, len) call will access 'len' bytes +- * from stack pointer 'buf'. Check it +- * note: regno == len, regno - 1 == buf ++ /* remember the mem_size which may be used later ++ * to refine return values. + */ +- if (regno == 0) { +- /* kernel subsystem misconfigured verifier */ +- verbose("ARG_CONST_STACK_SIZE cannot be first argument\n"); ++ meta->msize_max_value = reg->umax_value; ++ ++ /* The register is SCALAR_VALUE; the access check ++ * happens using its boundaries. ++ */ ++ if (!tnum_is_const(reg->var_off)) ++ /* For unprivileged variable accesses, disable raw ++ * mode so that the program is required to ++ * initialize all the memory that the helper could ++ * just partially fill up. ++ */ ++ meta = NULL; ++ ++ if (reg->smin_value < 0) { ++ verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", ++ regno); ++ return -EACCES; ++ } ++ ++ if (reg->umin_value == 0) { ++ err = check_helper_mem_access(env, regno - 1, 0, ++ zero_size_allowed, ++ meta); ++ if (err) ++ return err; ++ } ++ ++ if (reg->umax_value >= BPF_MAX_VAR_SIZ) { ++ verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", ++ regno); + return -EACCES; + } +- err = check_stack_boundary(env, regno - 1, reg->imm); ++ err = check_helper_mem_access(env, regno - 1, ++ reg->umax_value, ++ zero_size_allowed, meta); ++ if (!err) ++ err = mark_chain_precision(env, regno); ++ } else if (arg_type_is_int_ptr(arg_type)) { ++ int size = int_ptr_type_to_size(arg_type); ++ ++ err = check_helper_mem_access(env, regno, size, false, meta); ++ if (err) ++ return err; ++ err = check_ptr_alignment(env, reg, 0, size, true); + } + + return err; ++err_type: ++ verbose(env, "R%d type=%s expected=%s\n", regno, ++ reg_type_str[type], reg_type_str[expected_type]); ++ return -EACCES; + } + +-static int check_map_func_compatibility(struct bpf_map *map, int func_id) ++static int check_map_func_compatibility(struct bpf_verifier_env *env, ++ struct bpf_map *map, int func_id) + { + if (!map) + return 0; +@@ -911,7 +3474,74 @@ static int check_map_func_compatibility( + break; + case BPF_MAP_TYPE_PERF_EVENT_ARRAY: + if (func_id != BPF_FUNC_perf_event_read && +- func_id != BPF_FUNC_perf_event_output) ++ func_id != BPF_FUNC_perf_event_output && ++ func_id != BPF_FUNC_perf_event_read_value) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_STACK_TRACE: ++ if (func_id != BPF_FUNC_get_stackid) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_CGROUP_ARRAY: ++ if (func_id != BPF_FUNC_skb_under_cgroup && ++ func_id != BPF_FUNC_current_task_under_cgroup) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_CGROUP_STORAGE: ++ if (func_id != BPF_FUNC_get_local_storage) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_DEVMAP: ++ case BPF_MAP_TYPE_DEVMAP_HASH: ++ if (func_id != BPF_FUNC_redirect_map && ++ func_id != BPF_FUNC_map_lookup_elem) ++ goto error; ++ break; ++ /* Restrict bpf side of cpumap and xskmap, open when use-cases ++ * appear. ++ */ ++ case BPF_MAP_TYPE_CPUMAP: ++ if (func_id != BPF_FUNC_redirect_map) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_XSKMAP: ++ if (func_id != BPF_FUNC_redirect_map && ++ func_id != BPF_FUNC_map_lookup_elem) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_ARRAY_OF_MAPS: ++ case BPF_MAP_TYPE_HASH_OF_MAPS: ++ if (func_id != BPF_FUNC_map_lookup_elem) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_SOCKMAP: ++ if (func_id != BPF_FUNC_sk_redirect_map && ++ func_id != BPF_FUNC_sock_map_update && ++ func_id != BPF_FUNC_map_delete_elem && ++ func_id != BPF_FUNC_msg_redirect_map) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_SOCKHASH: ++ if (func_id != BPF_FUNC_sk_redirect_hash && ++ func_id != BPF_FUNC_sock_hash_update && ++ func_id != BPF_FUNC_map_delete_elem && ++ func_id != BPF_FUNC_msg_redirect_hash) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: ++ if (func_id != BPF_FUNC_sk_select_reuseport) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_QUEUE: ++ case BPF_MAP_TYPE_STACK: ++ if (func_id != BPF_FUNC_map_peek_elem && ++ func_id != BPF_FUNC_map_pop_elem && ++ func_id != BPF_FUNC_map_push_elem) ++ goto error; ++ break; ++ case BPF_MAP_TYPE_SK_STORAGE: ++ if (func_id != BPF_FUNC_sk_storage_get && ++ func_id != BPF_FUNC_sk_storage_delete) + goto error; + break; + default: +@@ -923,109 +3553,1579 @@ static int check_map_func_compatibility( + case BPF_FUNC_tail_call: + if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) + goto error; ++ if (env->subprog_cnt > 1) { ++ verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n"); ++ return -EINVAL; ++ } + break; + case BPF_FUNC_perf_event_read: + case BPF_FUNC_perf_event_output: ++ case BPF_FUNC_perf_event_read_value: + if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) + goto error; + break; ++ case BPF_FUNC_get_stackid: ++ if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) ++ goto error; ++ break; ++ case BPF_FUNC_current_task_under_cgroup: ++ case BPF_FUNC_skb_under_cgroup: ++ if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) ++ goto error; ++ break; ++ case BPF_FUNC_redirect_map: ++ if (map->map_type != BPF_MAP_TYPE_DEVMAP && ++ map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && ++ map->map_type != BPF_MAP_TYPE_CPUMAP && ++ map->map_type != BPF_MAP_TYPE_XSKMAP) ++ goto error; ++ break; ++ case BPF_FUNC_sk_redirect_map: ++ case BPF_FUNC_msg_redirect_map: ++ case BPF_FUNC_sock_map_update: ++ if (map->map_type != BPF_MAP_TYPE_SOCKMAP) ++ goto error; ++ break; ++ case BPF_FUNC_sk_redirect_hash: ++ case BPF_FUNC_msg_redirect_hash: ++ case BPF_FUNC_sock_hash_update: ++ if (map->map_type != BPF_MAP_TYPE_SOCKHASH) ++ goto error; ++ break; ++ case BPF_FUNC_get_local_storage: ++ if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && ++ map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) ++ goto error; ++ break; ++ case BPF_FUNC_sk_select_reuseport: ++ if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) ++ goto error; ++ break; ++ case BPF_FUNC_map_peek_elem: ++ case BPF_FUNC_map_pop_elem: ++ case BPF_FUNC_map_push_elem: ++ if (map->map_type != BPF_MAP_TYPE_QUEUE && ++ map->map_type != BPF_MAP_TYPE_STACK) ++ goto error; ++ break; ++ case BPF_FUNC_sk_storage_get: ++ case BPF_FUNC_sk_storage_delete: ++ if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) ++ goto error; ++ break; + default: + break; + } + + return 0; + error: +- verbose("cannot pass map_type %d into func %d\n", +- map->map_type, func_id); ++ verbose(env, "cannot pass map_type %d into func %s#%d\n", ++ map->map_type, func_id_name(func_id), func_id); + return -EINVAL; + } + +-static int check_call(struct verifier_env *env, int func_id) ++static bool check_raw_mode_ok(const struct bpf_func_proto *fn) ++{ ++ int count = 0; ++ ++ if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) ++ count++; ++ if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) ++ count++; ++ if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) ++ count++; ++ if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) ++ count++; ++ if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) ++ count++; ++ ++ /* We only support one arg being in raw mode at the moment, ++ * which is sufficient for the helper functions we have ++ * right now. ++ */ ++ return count <= 1; ++} ++ ++static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, ++ enum bpf_arg_type arg_next) ++{ ++ return (arg_type_is_mem_ptr(arg_curr) && ++ !arg_type_is_mem_size(arg_next)) || ++ (!arg_type_is_mem_ptr(arg_curr) && ++ arg_type_is_mem_size(arg_next)); ++} ++ ++static bool check_arg_pair_ok(const struct bpf_func_proto *fn) ++{ ++ /* bpf_xxx(..., buf, len) call will access 'len' ++ * bytes from memory 'buf'. Both arg types need ++ * to be paired, so make sure there's no buggy ++ * helper function specification. ++ */ ++ if (arg_type_is_mem_size(fn->arg1_type) || ++ arg_type_is_mem_ptr(fn->arg5_type) || ++ check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || ++ check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || ++ check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || ++ check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) ++ return false; ++ ++ return true; ++} ++ ++static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) ++{ ++ int count = 0; ++ ++ if (arg_type_may_be_refcounted(fn->arg1_type)) ++ count++; ++ if (arg_type_may_be_refcounted(fn->arg2_type)) ++ count++; ++ if (arg_type_may_be_refcounted(fn->arg3_type)) ++ count++; ++ if (arg_type_may_be_refcounted(fn->arg4_type)) ++ count++; ++ if (arg_type_may_be_refcounted(fn->arg5_type)) ++ count++; ++ ++ /* A reference acquiring function cannot acquire ++ * another refcounted ptr. ++ */ ++ if (is_acquire_function(func_id) && count) ++ return false; ++ ++ /* We only support one arg being unreferenced at the moment, ++ * which is sufficient for the helper functions we have right now. ++ */ ++ return count <= 1; ++} ++ ++static int check_func_proto(const struct bpf_func_proto *fn, int func_id) ++{ ++ return check_raw_mode_ok(fn) && ++ check_arg_pair_ok(fn) && ++ check_refcount_ok(fn, func_id) ? 0 : -EINVAL; ++} ++ ++/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] ++ * are now invalid, so turn them into unknown SCALAR_VALUE. ++ */ ++static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, ++ struct bpf_func_state *state) ++{ ++ struct bpf_reg_state *regs = state->regs, *reg; ++ int i; ++ ++ for (i = 0; i < MAX_BPF_REG; i++) ++ if (reg_is_pkt_pointer_any(®s[i])) ++ mark_reg_unknown(env, regs, i); ++ ++ bpf_for_each_spilled_reg(i, state, reg) { ++ if (!reg) ++ continue; ++ if (reg_is_pkt_pointer_any(reg)) ++ __mark_reg_unknown(env, reg); ++ } ++} ++ ++static void clear_all_pkt_pointers(struct bpf_verifier_env *env) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ int i; ++ ++ for (i = 0; i <= vstate->curframe; i++) ++ __clear_all_pkt_pointers(env, vstate->frame[i]); ++} ++ ++static void release_reg_references(struct bpf_verifier_env *env, ++ struct bpf_func_state *state, ++ int ref_obj_id) ++{ ++ struct bpf_reg_state *regs = state->regs, *reg; ++ int i; ++ ++ for (i = 0; i < MAX_BPF_REG; i++) ++ if (regs[i].ref_obj_id == ref_obj_id) ++ mark_reg_unknown(env, regs, i); ++ ++ bpf_for_each_spilled_reg(i, state, reg) { ++ if (!reg) ++ continue; ++ if (reg->ref_obj_id == ref_obj_id) ++ __mark_reg_unknown(env, reg); ++ } ++} ++ ++/* The pointer with the specified id has released its reference to kernel ++ * resources. Identify all copies of the same pointer and clear the reference. ++ */ ++static int release_reference(struct bpf_verifier_env *env, ++ int ref_obj_id) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ int err; ++ int i; ++ ++ err = release_reference_state(cur_func(env), ref_obj_id); ++ if (err) ++ return err; ++ ++ for (i = 0; i <= vstate->curframe; i++) ++ release_reg_references(env, vstate->frame[i], ref_obj_id); ++ ++ return 0; ++} ++ ++static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, ++ int *insn_idx) ++{ ++ struct bpf_verifier_state *state = env->cur_state; ++ struct bpf_func_state *caller, *callee; ++ int i, err, subprog, target_insn; ++ ++ if (state->curframe + 1 >= MAX_CALL_FRAMES) { ++ verbose(env, "the call stack of %d frames is too deep\n", ++ state->curframe + 2); ++ return -E2BIG; ++ } ++ ++ target_insn = *insn_idx + insn->imm; ++ subprog = find_subprog(env, target_insn + 1); ++ if (subprog < 0) { ++ verbose(env, "verifier bug. No program starts at insn %d\n", ++ target_insn + 1); ++ return -EFAULT; ++ } ++ ++ caller = state->frame[state->curframe]; ++ if (state->frame[state->curframe + 1]) { ++ verbose(env, "verifier bug. Frame %d already allocated\n", ++ state->curframe + 1); ++ return -EFAULT; ++ } ++ ++ callee = kzalloc(sizeof(*callee), GFP_KERNEL); ++ if (!callee) ++ return -ENOMEM; ++ state->frame[state->curframe + 1] = callee; ++ ++ /* callee cannot access r0, r6 - r9 for reading and has to write ++ * into its own stack before reading from it. ++ * callee can read/write into caller's stack ++ */ ++ init_func_state(env, callee, ++ /* remember the callsite, it will be used by bpf_exit */ ++ *insn_idx /* callsite */, ++ state->curframe + 1 /* frameno within this callchain */, ++ subprog /* subprog number within this prog */); ++ ++ /* Transfer references to the callee */ ++ err = transfer_reference_state(callee, caller); ++ if (err) ++ return err; ++ ++ /* copy r1 - r5 args that callee can access. The copy includes parent ++ * pointers, which connects us up to the liveness chain ++ */ ++ for (i = BPF_REG_1; i <= BPF_REG_5; i++) ++ callee->regs[i] = caller->regs[i]; ++ ++ /* after the call registers r0 - r5 were scratched */ ++ for (i = 0; i < CALLER_SAVED_REGS; i++) { ++ mark_reg_not_init(env, caller->regs, caller_saved[i]); ++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); ++ } ++ ++ /* only increment it after check_reg_arg() finished */ ++ state->curframe++; ++ ++ /* and go analyze first insn of the callee */ ++ *insn_idx = target_insn; ++ ++ if (env->log.level & BPF_LOG_LEVEL) { ++ verbose(env, "caller:\n"); ++ print_verifier_state(env, caller); ++ verbose(env, "callee:\n"); ++ print_verifier_state(env, callee); ++ } ++ return 0; ++} ++ ++static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) ++{ ++ struct bpf_verifier_state *state = env->cur_state; ++ struct bpf_func_state *caller, *callee; ++ struct bpf_reg_state *r0; ++ int err; ++ ++ callee = state->frame[state->curframe]; ++ r0 = &callee->regs[BPF_REG_0]; ++ if (r0->type == PTR_TO_STACK) { ++ /* technically it's ok to return caller's stack pointer ++ * (or caller's caller's pointer) back to the caller, ++ * since these pointers are valid. Only current stack ++ * pointer will be invalid as soon as function exits, ++ * but let's be conservative ++ */ ++ verbose(env, "cannot return stack pointer to the caller\n"); ++ return -EINVAL; ++ } ++ ++ state->curframe--; ++ caller = state->frame[state->curframe]; ++ /* return to the caller whatever r0 had in the callee */ ++ caller->regs[BPF_REG_0] = *r0; ++ ++ /* Transfer references to the caller */ ++ err = transfer_reference_state(caller, callee); ++ if (err) ++ return err; ++ ++ *insn_idx = callee->callsite + 1; ++ if (env->log.level & BPF_LOG_LEVEL) { ++ verbose(env, "returning from callee:\n"); ++ print_verifier_state(env, callee); ++ verbose(env, "to caller at %d:\n", *insn_idx); ++ print_verifier_state(env, caller); ++ } ++ /* clear everything in the callee */ ++ free_func_state(callee); ++ state->frame[state->curframe + 1] = NULL; ++ return 0; ++} ++ ++static int do_refine_retval_range(struct bpf_verifier_env *env, ++ struct bpf_reg_state *regs, int ret_type, ++ int func_id, struct bpf_call_arg_meta *meta) ++{ ++ struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; ++ struct bpf_reg_state tmp_reg = *ret_reg; ++ bool ret; ++ ++ if (ret_type != RET_INTEGER || ++ (func_id != BPF_FUNC_get_stack && ++ func_id != BPF_FUNC_probe_read_str)) ++ return 0; ++ ++ /* Error case where ret is in interval [S32MIN, -1]. */ ++ ret_reg->smin_value = S32_MIN; ++ ret_reg->smax_value = -1; ++ ++ __reg_deduce_bounds(ret_reg); ++ __reg_bound_offset(ret_reg); ++ __update_reg_bounds(ret_reg); ++ ++ ret = push_stack(env, env->insn_idx + 1, env->insn_idx, false); ++ if (!ret) ++ return -EFAULT; ++ ++ *ret_reg = tmp_reg; ++ ++ /* Success case where ret is in range [0, msize_max_value]. */ ++ ret_reg->smin_value = 0; ++ ret_reg->smax_value = meta->msize_max_value; ++ ret_reg->umin_value = ret_reg->smin_value; ++ ret_reg->umax_value = ret_reg->smax_value; ++ ++ __reg_deduce_bounds(ret_reg); ++ __reg_bound_offset(ret_reg); ++ __update_reg_bounds(ret_reg); ++ ++ return 0; ++} ++ ++static int ++record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, ++ int func_id, int insn_idx) ++{ ++ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; ++ struct bpf_map *map = meta->map_ptr; ++ ++ if (func_id != BPF_FUNC_tail_call && ++ func_id != BPF_FUNC_map_lookup_elem && ++ func_id != BPF_FUNC_map_update_elem && ++ func_id != BPF_FUNC_map_delete_elem && ++ func_id != BPF_FUNC_map_push_elem && ++ func_id != BPF_FUNC_map_pop_elem && ++ func_id != BPF_FUNC_map_peek_elem) ++ return 0; ++ ++ if (map == NULL) { ++ verbose(env, "kernel subsystem misconfigured verifier\n"); ++ return -EINVAL; ++ } ++ ++ /* In case of read-only, some additional restrictions ++ * need to be applied in order to prevent altering the ++ * state of the map from program side. ++ */ ++ if ((map->map_flags & BPF_F_RDONLY_PROG) && ++ (func_id == BPF_FUNC_map_delete_elem || ++ func_id == BPF_FUNC_map_update_elem || ++ func_id == BPF_FUNC_map_push_elem || ++ func_id == BPF_FUNC_map_pop_elem)) { ++ verbose(env, "write into map forbidden\n"); ++ return -EACCES; ++ } ++ ++ if (!BPF_MAP_PTR(aux->map_state)) ++ bpf_map_ptr_store(aux, meta->map_ptr, ++ meta->map_ptr->unpriv_array); ++ else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr) ++ bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, ++ meta->map_ptr->unpriv_array); ++ return 0; ++} ++ ++static int check_reference_leak(struct bpf_verifier_env *env) ++{ ++ struct bpf_func_state *state = cur_func(env); ++ int i; ++ ++ for (i = 0; i < state->acquired_refs; i++) { ++ verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", ++ state->refs[i].id, state->refs[i].insn_idx); ++ } ++ return state->acquired_refs ? -EINVAL : 0; ++} ++ ++static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) + { +- struct verifier_state *state = &env->cur_state; + const struct bpf_func_proto *fn = NULL; +- struct reg_state *regs = state->regs; +- struct bpf_map *map = NULL; +- struct reg_state *reg; ++ struct bpf_reg_state *regs; ++ struct bpf_call_arg_meta meta; ++ bool changes_data; + int i, err; + + /* find function prototype */ + if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { +- verbose("invalid func %d\n", func_id); ++ verbose(env, "invalid func %s#%d\n", func_id_name(func_id), ++ func_id); + return -EINVAL; + } + +- if (env->prog->aux->ops->get_func_proto) +- fn = env->prog->aux->ops->get_func_proto(func_id); +- ++ if (env->ops->get_func_proto) ++ fn = env->ops->get_func_proto(func_id, env->prog); + if (!fn) { +- verbose("unknown func %d\n", func_id); ++ verbose(env, "unknown func %s#%d\n", func_id_name(func_id), ++ func_id); + return -EINVAL; + } + + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + if (!env->prog->gpl_compatible && fn->gpl_only) { +- verbose("cannot call GPL only function from proprietary program\n"); ++ verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); ++ return -EINVAL; ++ } ++ ++ /* With LD_ABS/IND some JITs save/restore skb from r1. */ ++ changes_data = bpf_helper_changes_pkt_data(fn->func); ++ if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { ++ verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", ++ func_id_name(func_id), func_id); + return -EINVAL; + } + ++ memset(&meta, 0, sizeof(meta)); ++ meta.pkt_access = fn->pkt_access; ++ ++ err = check_func_proto(fn, func_id); ++ if (err) { ++ verbose(env, "kernel subsystem misconfigured func %s#%d\n", ++ func_id_name(func_id), func_id); ++ return err; ++ } ++ ++ meta.func_id = func_id; + /* check args */ +- err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &map); ++ err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta); + if (err) + return err; +- err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &map); ++ err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta); + if (err) + return err; +- err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &map); ++ err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta); + if (err) + return err; +- err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &map); ++ err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta); + if (err) + return err; +- err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &map); ++ err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta); + if (err) + return err; + ++ err = record_func_map(env, &meta, func_id, insn_idx); ++ if (err) ++ return err; ++ ++ /* Mark slots with STACK_MISC in case of raw mode, stack offset ++ * is inferred from register state. ++ */ ++ for (i = 0; i < meta.access_size; i++) { ++ err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, ++ BPF_WRITE, -1, false); ++ if (err) ++ return err; ++ } ++ ++ if (func_id == BPF_FUNC_tail_call) { ++ err = check_reference_leak(env); ++ if (err) { ++ verbose(env, "tail_call would lead to reference leak\n"); ++ return err; ++ } ++ } else if (is_release_function(func_id)) { ++ err = release_reference(env, meta.ref_obj_id); ++ if (err) { ++ verbose(env, "func %s#%d reference has not been acquired before\n", ++ func_id_name(func_id), func_id); ++ return err; ++ } ++ } ++ ++ regs = cur_regs(env); ++ ++ /* check that flags argument in get_local_storage(map, flags) is 0, ++ * this is required because get_local_storage() can't return an error. ++ */ ++ if (func_id == BPF_FUNC_get_local_storage && ++ !register_is_null(®s[BPF_REG_2])) { ++ verbose(env, "get_local_storage() doesn't support non-zero flags\n"); ++ return -EINVAL; ++ } ++ + /* reset caller saved regs */ + for (i = 0; i < CALLER_SAVED_REGS; i++) { +- reg = regs + caller_saved[i]; +- reg->type = NOT_INIT; +- reg->imm = 0; ++ mark_reg_not_init(env, regs, caller_saved[i]); ++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); + } + +- /* update return register */ ++ /* helper call returns 64-bit value. */ ++ regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; ++ ++ /* update return register (already marked as written above) */ + if (fn->ret_type == RET_INTEGER) { +- regs[BPF_REG_0].type = UNKNOWN_VALUE; ++ /* sets type to SCALAR_VALUE */ ++ mark_reg_unknown(env, regs, BPF_REG_0); + } else if (fn->ret_type == RET_VOID) { + regs[BPF_REG_0].type = NOT_INIT; +- } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL) { +- regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; ++ } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || ++ fn->ret_type == RET_PTR_TO_MAP_VALUE) { ++ /* There is no offset yet applied, variable or fixed */ ++ mark_reg_known_zero(env, regs, BPF_REG_0); + /* remember map_ptr, so that check_map_access() + * can check 'value_size' boundary of memory access + * to map element returned from bpf_map_lookup_elem() + */ +- if (map == NULL) { +- verbose("kernel subsystem misconfigured verifier\n"); ++ if (meta.map_ptr == NULL) { ++ verbose(env, ++ "kernel subsystem misconfigured verifier\n"); + return -EINVAL; + } +- regs[BPF_REG_0].map_ptr = map; ++ regs[BPF_REG_0].map_ptr = meta.map_ptr; ++ if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { ++ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; ++ if (map_value_has_spin_lock(meta.map_ptr)) ++ regs[BPF_REG_0].id = ++env->id_gen; ++ } else { ++ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; ++ regs[BPF_REG_0].id = ++env->id_gen; ++ } ++ } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { ++ mark_reg_known_zero(env, regs, BPF_REG_0); ++ regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; ++ regs[BPF_REG_0].id = ++env->id_gen; ++ } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { ++ mark_reg_known_zero(env, regs, BPF_REG_0); ++ regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; ++ regs[BPF_REG_0].id = ++env->id_gen; ++ } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { ++ mark_reg_known_zero(env, regs, BPF_REG_0); ++ regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; ++ regs[BPF_REG_0].id = ++env->id_gen; + } else { +- verbose("unknown return type %d of func %d\n", +- fn->ret_type, func_id); ++ verbose(env, "unknown return type %d of func %s#%d\n", ++ fn->ret_type, func_id_name(func_id), func_id); + return -EINVAL; + } + +- err = check_map_func_compatibility(map, func_id); ++ if (is_ptr_cast_function(func_id)) { ++ /* For release_reference() */ ++ regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; ++ } else if (is_acquire_function(func_id)) { ++ int id = acquire_reference_state(env, insn_idx); ++ ++ if (id < 0) ++ return id; ++ /* For mark_ptr_or_null_reg() */ ++ regs[BPF_REG_0].id = id; ++ /* For release_reference() */ ++ regs[BPF_REG_0].ref_obj_id = id; ++ } ++ ++ err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); ++ if (err) ++ return err; ++ ++ err = check_map_func_compatibility(env, meta.map_ptr, func_id); + if (err) + return err; + ++ if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) { ++ const char *err_str; ++ ++#ifdef CONFIG_PERF_EVENTS ++ err = get_callchain_buffers(sysctl_perf_event_max_stack); ++ err_str = "cannot get callchain buffer for func %s#%d\n"; ++#else ++ err = -ENOTSUPP; ++ err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; ++#endif ++ if (err) { ++ verbose(env, err_str, func_id_name(func_id), func_id); ++ return err; ++ } ++ ++ env->prog->has_callchain_buf = true; ++ } ++ ++ if (changes_data) ++ clear_all_pkt_pointers(env); + return 0; + } + ++static bool signed_add_overflows(s64 a, s64 b) ++{ ++ /* Do the add in u64, where overflow is well-defined */ ++ s64 res = (s64)((u64)a + (u64)b); ++ ++ if (b < 0) ++ return res > a; ++ return res < a; ++} ++ ++static bool signed_sub_overflows(s64 a, s64 b) ++{ ++ /* Do the sub in u64, where overflow is well-defined */ ++ s64 res = (s64)((u64)a - (u64)b); ++ ++ if (b < 0) ++ return res < a; ++ return res > a; ++} ++ ++static bool check_reg_sane_offset(struct bpf_verifier_env *env, ++ const struct bpf_reg_state *reg, ++ enum bpf_reg_type type) ++{ ++ bool known = tnum_is_const(reg->var_off); ++ s64 val = reg->var_off.value; ++ s64 smin = reg->smin_value; ++ ++ if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { ++ verbose(env, "math between %s pointer and %lld is not allowed\n", ++ reg_type_str[type], val); ++ return false; ++ } ++ ++ if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { ++ verbose(env, "%s pointer offset %d is not allowed\n", ++ reg_type_str[type], reg->off); ++ return false; ++ } ++ ++ if (smin == S64_MIN) { ++ verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", ++ reg_type_str[type]); ++ return false; ++ } ++ ++ if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { ++ verbose(env, "value %lld makes %s pointer be out of bounds\n", ++ smin, reg_type_str[type]); ++ return false; ++ } ++ ++ return true; ++} ++ ++static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) ++{ ++ return &env->insn_aux_data[env->insn_idx]; ++} ++ ++enum { ++ REASON_BOUNDS = -1, ++ REASON_TYPE = -2, ++ REASON_PATHS = -3, ++ REASON_LIMIT = -4, ++ REASON_STACK = -5, ++}; ++ ++static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, ++ u32 *alu_limit, bool mask_to_left) ++{ ++ u32 max = 0, ptr_limit = 0; ++ ++ switch (ptr_reg->type) { ++ case PTR_TO_STACK: ++ /* Offset 0 is out-of-bounds, but acceptable start for the ++ * left direction, see BPF_REG_FP. Also, unknown scalar ++ * offset where we would need to deal with min/max bounds is ++ * currently prohibited for unprivileged. ++ */ ++ max = MAX_BPF_STACK + mask_to_left; ++ ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); ++ break; ++ case PTR_TO_MAP_VALUE: ++ max = ptr_reg->map_ptr->value_size; ++ ptr_limit = (mask_to_left ? ++ ptr_reg->smin_value : ++ ptr_reg->umax_value) + ptr_reg->off; ++ break; ++ default: ++ return REASON_TYPE; ++ } ++ ++ if (ptr_limit >= max) ++ return REASON_LIMIT; ++ *alu_limit = ptr_limit; ++ return 0; ++} ++ ++static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, ++ const struct bpf_insn *insn) ++{ ++ return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K; ++} ++ ++static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, ++ u32 alu_state, u32 alu_limit) ++{ ++ /* If we arrived here from different branches with different ++ * state or limits to sanitize, then this won't work. ++ */ ++ if (aux->alu_state && ++ (aux->alu_state != alu_state || ++ aux->alu_limit != alu_limit)) ++ return REASON_PATHS; ++ ++ /* Corresponding fixup done in fixup_bpf_calls(). */ ++ aux->alu_state = alu_state; ++ aux->alu_limit = alu_limit; ++ return 0; ++} ++ ++static int sanitize_val_alu(struct bpf_verifier_env *env, ++ struct bpf_insn *insn) ++{ ++ struct bpf_insn_aux_data *aux = cur_aux(env); ++ ++ if (can_skip_alu_sanitation(env, insn)) ++ return 0; ++ ++ return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); ++} ++ ++static bool sanitize_needed(u8 opcode) ++{ ++ return opcode == BPF_ADD || opcode == BPF_SUB; ++} ++ ++struct bpf_sanitize_info { ++ struct bpf_insn_aux_data aux; ++ bool mask_to_left; ++}; ++ ++static struct bpf_verifier_state * ++sanitize_speculative_path(struct bpf_verifier_env *env, ++ const struct bpf_insn *insn, ++ u32 next_idx, u32 curr_idx) ++{ ++ struct bpf_verifier_state *branch; ++ struct bpf_reg_state *regs; ++ ++ branch = push_stack(env, next_idx, curr_idx, true); ++ if (branch && insn) { ++ regs = branch->frame[branch->curframe]->regs; ++ if (BPF_SRC(insn->code) == BPF_K) { ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ } else if (BPF_SRC(insn->code) == BPF_X) { ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ mark_reg_unknown(env, regs, insn->src_reg); ++ } ++ } ++ return branch; ++} ++ ++static int sanitize_ptr_alu(struct bpf_verifier_env *env, ++ struct bpf_insn *insn, ++ const struct bpf_reg_state *ptr_reg, ++ const struct bpf_reg_state *off_reg, ++ struct bpf_reg_state *dst_reg, ++ struct bpf_sanitize_info *info, ++ const bool commit_window) ++{ ++ struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; ++ struct bpf_verifier_state *vstate = env->cur_state; ++ bool off_is_imm = tnum_is_const(off_reg->var_off); ++ bool off_is_neg = off_reg->smin_value < 0; ++ bool ptr_is_dst_reg = ptr_reg == dst_reg; ++ u8 opcode = BPF_OP(insn->code); ++ u32 alu_state, alu_limit; ++ struct bpf_reg_state tmp; ++ bool ret; ++ int err; ++ ++ if (can_skip_alu_sanitation(env, insn)) ++ return 0; ++ ++ /* We already marked aux for masking from non-speculative ++ * paths, thus we got here in the first place. We only care ++ * to explore bad access from here. ++ */ ++ if (vstate->speculative) ++ goto do_sim; ++ ++ if (!commit_window) { ++ if (!tnum_is_const(off_reg->var_off) && ++ (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) ++ return REASON_BOUNDS; ++ ++ info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || ++ (opcode == BPF_SUB && !off_is_neg); ++ } ++ ++ err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); ++ if (err < 0) ++ return err; ++ ++ if (commit_window) { ++ /* In commit phase we narrow the masking window based on ++ * the observed pointer move after the simulated operation. ++ */ ++ alu_state = info->aux.alu_state; ++ alu_limit = abs(info->aux.alu_limit - alu_limit); ++ } else { ++ alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; ++ alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; ++ alu_state |= ptr_is_dst_reg ? ++ BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; ++ } ++ ++ err = update_alu_sanitation_state(aux, alu_state, alu_limit); ++ if (err < 0) ++ return err; ++do_sim: ++ /* If we're in commit phase, we're done here given we already ++ * pushed the truncated dst_reg into the speculative verification ++ * stack. ++ * ++ * Also, when register is a known constant, we rewrite register-based ++ * operation to immediate-based, and thus do not need masking (and as ++ * a consequence, do not need to simulate the zero-truncation either). ++ */ ++ if (commit_window || off_is_imm) ++ return 0; ++ ++ /* Simulate and find potential out-of-bounds access under ++ * speculative execution from truncation as a result of ++ * masking when off was not within expected range. If off ++ * sits in dst, then we temporarily need to move ptr there ++ * to simulate dst (== 0) +/-= ptr. Needed, for example, ++ * for cases where we use K-based arithmetic in one direction ++ * and truncated reg-based in the other in order to explore ++ * bad access. ++ */ ++ if (!ptr_is_dst_reg) { ++ tmp = *dst_reg; ++ *dst_reg = *ptr_reg; ++ } ++ ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, ++ env->insn_idx); ++ if (!ptr_is_dst_reg && ret) ++ *dst_reg = tmp; ++ return !ret ? REASON_STACK : 0; ++} ++ ++static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ ++ /* If we simulate paths under speculation, we don't update the ++ * insn as 'seen' such that when we verify unreachable paths in ++ * the non-speculative domain, sanitize_dead_code() can still ++ * rewrite/sanitize them. ++ */ ++ if (!vstate->speculative) ++ env->insn_aux_data[env->insn_idx].seen = true; ++} ++ ++static int sanitize_err(struct bpf_verifier_env *env, ++ const struct bpf_insn *insn, int reason, ++ const struct bpf_reg_state *off_reg, ++ const struct bpf_reg_state *dst_reg) ++{ ++ static const char *err = "pointer arithmetic with it prohibited for !root"; ++ const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; ++ u32 dst = insn->dst_reg, src = insn->src_reg; ++ ++ switch (reason) { ++ case REASON_BOUNDS: ++ verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", ++ off_reg == dst_reg ? dst : src, err); ++ break; ++ case REASON_TYPE: ++ verbose(env, "R%d has pointer with unsupported alu operation, %s\n", ++ off_reg == dst_reg ? src : dst, err); ++ break; ++ case REASON_PATHS: ++ verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", ++ dst, op, err); ++ break; ++ case REASON_LIMIT: ++ verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", ++ dst, op, err); ++ break; ++ case REASON_STACK: ++ verbose(env, "R%d could not be pushed for speculative verification, %s\n", ++ dst, err); ++ break; ++ default: ++ verbose(env, "verifier internal error: unknown reason (%d)\n", ++ reason); ++ break; ++ } ++ ++ return -EACCES; ++} ++ ++static int sanitize_check_bounds(struct bpf_verifier_env *env, ++ const struct bpf_insn *insn, ++ const struct bpf_reg_state *dst_reg) ++{ ++ u32 dst = insn->dst_reg; ++ ++ /* For unprivileged we require that resulting offset must be in bounds ++ * in order to be able to sanitize access later on. ++ */ ++ if (env->allow_ptr_leaks) ++ return 0; ++ ++ switch (dst_reg->type) { ++ case PTR_TO_STACK: ++ if (check_stack_access(env, dst_reg, dst_reg->off + ++ dst_reg->var_off.value, 1)) { ++ verbose(env, "R%d stack pointer arithmetic goes out of range, " ++ "prohibited for !root\n", dst); ++ return -EACCES; ++ } ++ break; ++ case PTR_TO_MAP_VALUE: ++ if (check_map_access(env, dst, dst_reg->off, 1, false)) { ++ verbose(env, "R%d pointer arithmetic of map value goes out of range, " ++ "prohibited for !root\n", dst); ++ return -EACCES; ++ } ++ break; ++ default: ++ break; ++ } ++ ++ return 0; ++} ++ ++/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. ++ * Caller should also handle BPF_MOV case separately. ++ * If we return -EACCES, caller may want to try again treating pointer as a ++ * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. ++ */ ++static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, ++ struct bpf_insn *insn, ++ const struct bpf_reg_state *ptr_reg, ++ const struct bpf_reg_state *off_reg) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ struct bpf_reg_state *regs = state->regs, *dst_reg; ++ bool known = tnum_is_const(off_reg->var_off); ++ s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, ++ smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; ++ u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, ++ umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; ++ struct bpf_sanitize_info info = {}; ++ u8 opcode = BPF_OP(insn->code); ++ u32 dst = insn->dst_reg; ++ int ret; ++ ++ dst_reg = ®s[dst]; ++ ++ if ((known && (smin_val != smax_val || umin_val != umax_val)) || ++ smin_val > smax_val || umin_val > umax_val) { ++ /* Taint dst register if offset had invalid bounds derived from ++ * e.g. dead branches. ++ */ ++ __mark_reg_unknown(env, dst_reg); ++ return 0; ++ } ++ ++ if (BPF_CLASS(insn->code) != BPF_ALU64) { ++ /* 32-bit ALU ops on pointers produce (meaningless) scalars */ ++ verbose(env, ++ "R%d 32-bit pointer arithmetic prohibited\n", ++ dst); ++ return -EACCES; ++ } ++ ++ switch (ptr_reg->type) { ++ case PTR_TO_MAP_VALUE_OR_NULL: ++ verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", ++ dst, reg_type_str[ptr_reg->type]); ++ return -EACCES; ++ case CONST_PTR_TO_MAP: ++ /* smin_val represents the known value */ ++ if (known && smin_val == 0 && opcode == BPF_ADD) ++ break; ++ /* fall-through */ ++ case PTR_TO_PACKET_END: ++ case PTR_TO_SOCKET: ++ case PTR_TO_SOCKET_OR_NULL: ++ case PTR_TO_SOCK_COMMON: ++ case PTR_TO_SOCK_COMMON_OR_NULL: ++ case PTR_TO_TCP_SOCK: ++ case PTR_TO_TCP_SOCK_OR_NULL: ++ case PTR_TO_XDP_SOCK: ++ verbose(env, "R%d pointer arithmetic on %s prohibited\n", ++ dst, reg_type_str[ptr_reg->type]); ++ return -EACCES; ++ default: ++ break; ++ } ++ ++ /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. ++ * The id may be overwritten later if we create a new variable offset. ++ */ ++ dst_reg->type = ptr_reg->type; ++ dst_reg->id = ptr_reg->id; ++ ++ if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || ++ !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) ++ return -EINVAL; ++ ++ if (sanitize_needed(opcode)) { ++ ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, ++ &info, false); ++ if (ret < 0) ++ return sanitize_err(env, insn, ret, off_reg, dst_reg); ++ } ++ ++ switch (opcode) { ++ case BPF_ADD: ++ /* We can take a fixed offset as long as it doesn't overflow ++ * the s32 'off' field ++ */ ++ if (known && (ptr_reg->off + smin_val == ++ (s64)(s32)(ptr_reg->off + smin_val))) { ++ /* pointer += K. Accumulate it into fixed offset */ ++ dst_reg->smin_value = smin_ptr; ++ dst_reg->smax_value = smax_ptr; ++ dst_reg->umin_value = umin_ptr; ++ dst_reg->umax_value = umax_ptr; ++ dst_reg->var_off = ptr_reg->var_off; ++ dst_reg->off = ptr_reg->off + smin_val; ++ dst_reg->raw = ptr_reg->raw; ++ break; ++ } ++ /* A new variable offset is created. Note that off_reg->off ++ * == 0, since it's a scalar. ++ * dst_reg gets the pointer type and since some positive ++ * integer value was added to the pointer, give it a new 'id' ++ * if it's a PTR_TO_PACKET. ++ * this creates a new 'base' pointer, off_reg (variable) gets ++ * added into the variable offset, and we copy the fixed offset ++ * from ptr_reg. ++ */ ++ if (signed_add_overflows(smin_ptr, smin_val) || ++ signed_add_overflows(smax_ptr, smax_val)) { ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ dst_reg->smin_value = smin_ptr + smin_val; ++ dst_reg->smax_value = smax_ptr + smax_val; ++ } ++ if (umin_ptr + umin_val < umin_ptr || ++ umax_ptr + umax_val < umax_ptr) { ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ } else { ++ dst_reg->umin_value = umin_ptr + umin_val; ++ dst_reg->umax_value = umax_ptr + umax_val; ++ } ++ dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); ++ dst_reg->off = ptr_reg->off; ++ dst_reg->raw = ptr_reg->raw; ++ if (reg_is_pkt_pointer(ptr_reg)) { ++ dst_reg->id = ++env->id_gen; ++ /* something was added to pkt_ptr, set range to zero */ ++ dst_reg->raw = 0; ++ } ++ break; ++ case BPF_SUB: ++ if (dst_reg == off_reg) { ++ /* scalar -= pointer. Creates an unknown scalar */ ++ verbose(env, "R%d tried to subtract pointer from scalar\n", ++ dst); ++ return -EACCES; ++ } ++ /* We don't allow subtraction from FP, because (according to ++ * test_verifier.c test "invalid fp arithmetic", JITs might not ++ * be able to deal with it. ++ */ ++ if (ptr_reg->type == PTR_TO_STACK) { ++ verbose(env, "R%d subtraction from stack pointer prohibited\n", ++ dst); ++ return -EACCES; ++ } ++ if (known && (ptr_reg->off - smin_val == ++ (s64)(s32)(ptr_reg->off - smin_val))) { ++ /* pointer -= K. Subtract it from fixed offset */ ++ dst_reg->smin_value = smin_ptr; ++ dst_reg->smax_value = smax_ptr; ++ dst_reg->umin_value = umin_ptr; ++ dst_reg->umax_value = umax_ptr; ++ dst_reg->var_off = ptr_reg->var_off; ++ dst_reg->id = ptr_reg->id; ++ dst_reg->off = ptr_reg->off - smin_val; ++ dst_reg->raw = ptr_reg->raw; ++ break; ++ } ++ /* A new variable offset is created. If the subtrahend is known ++ * nonnegative, then any reg->range we had before is still good. ++ */ ++ if (signed_sub_overflows(smin_ptr, smax_val) || ++ signed_sub_overflows(smax_ptr, smin_val)) { ++ /* Overflow possible, we know nothing */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ dst_reg->smin_value = smin_ptr - smax_val; ++ dst_reg->smax_value = smax_ptr - smin_val; ++ } ++ if (umin_ptr < umax_val) { ++ /* Overflow possible, we know nothing */ ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ } else { ++ /* Cannot overflow (as long as bounds are consistent) */ ++ dst_reg->umin_value = umin_ptr - umax_val; ++ dst_reg->umax_value = umax_ptr - umin_val; ++ } ++ dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); ++ dst_reg->off = ptr_reg->off; ++ dst_reg->raw = ptr_reg->raw; ++ if (reg_is_pkt_pointer(ptr_reg)) { ++ dst_reg->id = ++env->id_gen; ++ /* something was added to pkt_ptr, set range to zero */ ++ if (smin_val < 0) ++ dst_reg->raw = 0; ++ } ++ break; ++ case BPF_AND: ++ case BPF_OR: ++ case BPF_XOR: ++ /* bitwise ops on pointers are troublesome, prohibit. */ ++ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", ++ dst, bpf_alu_string[opcode >> 4]); ++ return -EACCES; ++ default: ++ /* other operators (e.g. MUL,LSH) produce non-pointer results */ ++ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", ++ dst, bpf_alu_string[opcode >> 4]); ++ return -EACCES; ++ } ++ ++ if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) ++ return -EINVAL; ++ ++ __update_reg_bounds(dst_reg); ++ __reg_deduce_bounds(dst_reg); ++ __reg_bound_offset(dst_reg); ++ ++ if (sanitize_check_bounds(env, insn, dst_reg) < 0) ++ return -EACCES; ++ if (sanitize_needed(opcode)) { ++ ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, ++ &info, true); ++ if (ret < 0) ++ return sanitize_err(env, insn, ret, off_reg, dst_reg); ++ } ++ ++ return 0; ++} ++ ++/* WARNING: This function does calculations on 64-bit values, but the actual ++ * execution may occur on 32-bit values. Therefore, things like bitshifts ++ * need extra checks in the 32-bit case. ++ */ ++static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, ++ struct bpf_insn *insn, ++ struct bpf_reg_state *dst_reg, ++ struct bpf_reg_state src_reg) ++{ ++ struct bpf_reg_state *regs = cur_regs(env); ++ u8 opcode = BPF_OP(insn->code); ++ bool src_known, dst_known; ++ s64 smin_val, smax_val; ++ u64 umin_val, umax_val; ++ u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; ++ int ret; ++ ++ if (insn_bitness == 32) { ++ /* Relevant for 32-bit RSH: Information can propagate towards ++ * LSB, so it isn't sufficient to only truncate the output to ++ * 32 bits. ++ */ ++ coerce_reg_to_size(dst_reg, 4); ++ coerce_reg_to_size(&src_reg, 4); ++ } ++ ++ smin_val = src_reg.smin_value; ++ smax_val = src_reg.smax_value; ++ umin_val = src_reg.umin_value; ++ umax_val = src_reg.umax_value; ++ src_known = tnum_is_const(src_reg.var_off); ++ dst_known = tnum_is_const(dst_reg->var_off); ++ ++ if ((src_known && (smin_val != smax_val || umin_val != umax_val)) || ++ smin_val > smax_val || umin_val > umax_val) { ++ /* Taint dst register if offset had invalid bounds derived from ++ * e.g. dead branches. ++ */ ++ __mark_reg_unknown(env, dst_reg); ++ return 0; ++ } ++ ++ if (!src_known && ++ opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { ++ __mark_reg_unknown(env, dst_reg); ++ return 0; ++ } ++ ++ if (sanitize_needed(opcode)) { ++ ret = sanitize_val_alu(env, insn); ++ if (ret < 0) ++ return sanitize_err(env, insn, ret, NULL, NULL); ++ } ++ ++ switch (opcode) { ++ case BPF_ADD: ++ if (signed_add_overflows(dst_reg->smin_value, smin_val) || ++ signed_add_overflows(dst_reg->smax_value, smax_val)) { ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ dst_reg->smin_value += smin_val; ++ dst_reg->smax_value += smax_val; ++ } ++ if (dst_reg->umin_value + umin_val < umin_val || ++ dst_reg->umax_value + umax_val < umax_val) { ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ } else { ++ dst_reg->umin_value += umin_val; ++ dst_reg->umax_value += umax_val; ++ } ++ dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); ++ break; ++ case BPF_SUB: ++ if (signed_sub_overflows(dst_reg->smin_value, smax_val) || ++ signed_sub_overflows(dst_reg->smax_value, smin_val)) { ++ /* Overflow possible, we know nothing */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ dst_reg->smin_value -= smax_val; ++ dst_reg->smax_value -= smin_val; ++ } ++ if (dst_reg->umin_value < umax_val) { ++ /* Overflow possible, we know nothing */ ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ } else { ++ /* Cannot overflow (as long as bounds are consistent) */ ++ dst_reg->umin_value -= umax_val; ++ dst_reg->umax_value -= umin_val; ++ } ++ dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); ++ break; ++ case BPF_MUL: ++ dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); ++ if (smin_val < 0 || dst_reg->smin_value < 0) { ++ /* Ain't nobody got time to multiply that sign */ ++ __mark_reg_unbounded(dst_reg); ++ __update_reg_bounds(dst_reg); ++ break; ++ } ++ /* Both values are positive, so we can work with unsigned and ++ * copy the result to signed (unless it exceeds S64_MAX). ++ */ ++ if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { ++ /* Potential overflow, we know nothing */ ++ __mark_reg_unbounded(dst_reg); ++ /* (except what we can learn from the var_off) */ ++ __update_reg_bounds(dst_reg); ++ break; ++ } ++ dst_reg->umin_value *= umin_val; ++ dst_reg->umax_value *= umax_val; ++ if (dst_reg->umax_value > S64_MAX) { ++ /* Overflow possible, we know nothing */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ dst_reg->smin_value = dst_reg->umin_value; ++ dst_reg->smax_value = dst_reg->umax_value; ++ } ++ break; ++ case BPF_AND: ++ if (src_known && dst_known) { ++ __mark_reg_known(dst_reg, dst_reg->var_off.value & ++ src_reg.var_off.value); ++ break; ++ } ++ /* We get our minimum from the var_off, since that's inherently ++ * bitwise. Our maximum is the minimum of the operands' maxima. ++ */ ++ dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); ++ dst_reg->umin_value = dst_reg->var_off.value; ++ dst_reg->umax_value = min(dst_reg->umax_value, umax_val); ++ if (dst_reg->smin_value < 0 || smin_val < 0) { ++ /* Lose signed bounds when ANDing negative numbers, ++ * ain't nobody got time for that. ++ */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ /* ANDing two positives gives a positive, so safe to ++ * cast result into s64. ++ */ ++ dst_reg->smin_value = dst_reg->umin_value; ++ dst_reg->smax_value = dst_reg->umax_value; ++ } ++ /* We may learn something more from the var_off */ ++ __update_reg_bounds(dst_reg); ++ break; ++ case BPF_OR: ++ if (src_known && dst_known) { ++ __mark_reg_known(dst_reg, dst_reg->var_off.value | ++ src_reg.var_off.value); ++ break; ++ } ++ /* We get our maximum from the var_off, and our minimum is the ++ * maximum of the operands' minima ++ */ ++ dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); ++ dst_reg->umin_value = max(dst_reg->umin_value, umin_val); ++ dst_reg->umax_value = dst_reg->var_off.value | ++ dst_reg->var_off.mask; ++ if (dst_reg->smin_value < 0 || smin_val < 0) { ++ /* Lose signed bounds when ORing negative numbers, ++ * ain't nobody got time for that. ++ */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ } else { ++ /* ORing two positives gives a positive, so safe to ++ * cast result into s64. ++ */ ++ dst_reg->smin_value = dst_reg->umin_value; ++ dst_reg->smax_value = dst_reg->umax_value; ++ } ++ /* We may learn something more from the var_off */ ++ __update_reg_bounds(dst_reg); ++ break; ++ case BPF_LSH: ++ if (umax_val >= insn_bitness) { ++ /* Shifts greater than 31 or 63 are undefined. ++ * This includes shifts by a negative number. ++ */ ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ break; ++ } ++ /* We lose all sign bit information (except what we can pick ++ * up from var_off) ++ */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ /* If we might shift our top bit out, then we know nothing */ ++ if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ } else { ++ dst_reg->umin_value <<= umin_val; ++ dst_reg->umax_value <<= umax_val; ++ } ++ dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); ++ /* We may learn something more from the var_off */ ++ __update_reg_bounds(dst_reg); ++ break; ++ case BPF_RSH: ++ if (umax_val >= insn_bitness) { ++ /* Shifts greater than 31 or 63 are undefined. ++ * This includes shifts by a negative number. ++ */ ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ break; ++ } ++ /* BPF_RSH is an unsigned shift. If the value in dst_reg might ++ * be negative, then either: ++ * 1) src_reg might be zero, so the sign bit of the result is ++ * unknown, so we lose our signed bounds ++ * 2) it's known negative, thus the unsigned bounds capture the ++ * signed bounds ++ * 3) the signed bounds cross zero, so they tell us nothing ++ * about the result ++ * If the value in dst_reg is known nonnegative, then again the ++ * unsigned bounts capture the signed bounds. ++ * Thus, in all cases it suffices to blow away our signed bounds ++ * and rely on inferring new ones from the unsigned bounds and ++ * var_off of the result. ++ */ ++ dst_reg->smin_value = S64_MIN; ++ dst_reg->smax_value = S64_MAX; ++ dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); ++ dst_reg->umin_value >>= umax_val; ++ dst_reg->umax_value >>= umin_val; ++ /* We may learn something more from the var_off */ ++ __update_reg_bounds(dst_reg); ++ break; ++ case BPF_ARSH: ++ if (umax_val >= insn_bitness) { ++ /* Shifts greater than 31 or 63 are undefined. ++ * This includes shifts by a negative number. ++ */ ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ break; ++ } ++ ++ /* Upon reaching here, src_known is true and ++ * umax_val is equal to umin_val. ++ */ ++ if (insn_bitness == 32) { ++ dst_reg->smin_value = (u32)(((s32)dst_reg->smin_value) >> umin_val); ++ dst_reg->smax_value = (u32)(((s32)dst_reg->smax_value) >> umin_val); ++ } else { ++ dst_reg->smin_value >>= umin_val; ++ dst_reg->smax_value >>= umin_val; ++ } ++ ++ dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, ++ insn_bitness); ++ ++ /* blow away the dst_reg umin_value/umax_value and rely on ++ * dst_reg var_off to refine the result. ++ */ ++ dst_reg->umin_value = 0; ++ dst_reg->umax_value = U64_MAX; ++ __update_reg_bounds(dst_reg); ++ break; ++ default: ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ break; ++ } ++ ++ if (BPF_CLASS(insn->code) != BPF_ALU64) { ++ /* 32-bit ALU ops are (32,32)->32 */ ++ coerce_reg_to_size(dst_reg, 4); ++ } ++ ++ __reg_deduce_bounds(dst_reg); ++ __reg_bound_offset(dst_reg); ++ return 0; ++} ++ ++/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max ++ * and var_off. ++ */ ++static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, ++ struct bpf_insn *insn) ++{ ++ struct bpf_verifier_state *vstate = env->cur_state; ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; ++ struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; ++ u8 opcode = BPF_OP(insn->code); ++ int err; ++ ++ dst_reg = ®s[insn->dst_reg]; ++ src_reg = NULL; ++ if (dst_reg->type != SCALAR_VALUE) ++ ptr_reg = dst_reg; ++ if (BPF_SRC(insn->code) == BPF_X) { ++ src_reg = ®s[insn->src_reg]; ++ if (src_reg->type != SCALAR_VALUE) { ++ if (dst_reg->type != SCALAR_VALUE) { ++ /* Combining two pointers by any ALU op yields ++ * an arbitrary scalar. Disallow all math except ++ * pointer subtraction ++ */ ++ if (opcode == BPF_SUB && env->allow_ptr_leaks) { ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ return 0; ++ } ++ verbose(env, "R%d pointer %s pointer prohibited\n", ++ insn->dst_reg, ++ bpf_alu_string[opcode >> 4]); ++ return -EACCES; ++ } else { ++ /* scalar += pointer ++ * This is legal, but we have to reverse our ++ * src/dest handling in computing the range ++ */ ++ err = mark_chain_precision(env, insn->dst_reg); ++ if (err) ++ return err; ++ return adjust_ptr_min_max_vals(env, insn, ++ src_reg, dst_reg); ++ } ++ } else if (ptr_reg) { ++ /* pointer += scalar */ ++ err = mark_chain_precision(env, insn->src_reg); ++ if (err) ++ return err; ++ return adjust_ptr_min_max_vals(env, insn, ++ dst_reg, src_reg); ++ } ++ } else { ++ /* Pretend the src is a reg with a known value, since we only ++ * need to be able to read from this state. ++ */ ++ off_reg.type = SCALAR_VALUE; ++ __mark_reg_known(&off_reg, insn->imm); ++ src_reg = &off_reg; ++ if (ptr_reg) /* pointer += K */ ++ return adjust_ptr_min_max_vals(env, insn, ++ ptr_reg, src_reg); ++ } ++ ++ /* Got here implies adding two SCALAR_VALUEs */ ++ if (WARN_ON_ONCE(ptr_reg)) { ++ print_verifier_state(env, state); ++ verbose(env, "verifier internal error: unexpected ptr_reg\n"); ++ return -EINVAL; ++ } ++ if (WARN_ON(!src_reg)) { ++ print_verifier_state(env, state); ++ verbose(env, "verifier internal error: no src_reg\n"); ++ return -EINVAL; ++ } ++ return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); ++} ++ + /* check validity of 32-bit and 64-bit arithmetic operations */ +-static int check_alu_op(struct verifier_env *env, struct bpf_insn *insn) ++static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) + { +- struct reg_state *regs = env->cur_state.regs; ++ struct bpf_reg_state *regs = cur_regs(env); + u8 opcode = BPF_OP(insn->code); + int err; + +@@ -1034,30 +5134,31 @@ static int check_alu_op(struct verifier_ + if (BPF_SRC(insn->code) != 0 || + insn->src_reg != BPF_REG_0 || + insn->off != 0 || insn->imm != 0) { +- verbose("BPF_NEG uses reserved fields\n"); ++ verbose(env, "BPF_NEG uses reserved fields\n"); + return -EINVAL; + } + } else { + if (insn->src_reg != BPF_REG_0 || insn->off != 0 || +- (insn->imm != 16 && insn->imm != 32 && insn->imm != 64)) { +- verbose("BPF_END uses reserved fields\n"); ++ (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || ++ BPF_CLASS(insn->code) == BPF_ALU64) { ++ verbose(env, "BPF_END uses reserved fields\n"); + return -EINVAL; + } + } + + /* check src operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + if (is_pointer_value(env, insn->dst_reg)) { +- verbose("R%d pointer arithmetic prohibited\n", ++ verbose(env, "R%d pointer arithmetic prohibited\n", + insn->dst_reg); + return -EACCES; + } + + /* check dest operand */ +- err = check_reg_arg(regs, insn->dst_reg, DST_OP); ++ err = check_reg_arg(env, insn->dst_reg, DST_OP); + if (err) + return err; + +@@ -1065,81 +5166,100 @@ static int check_alu_op(struct verifier_ + + if (BPF_SRC(insn->code) == BPF_X) { + if (insn->imm != 0 || insn->off != 0) { +- verbose("BPF_MOV uses reserved fields\n"); ++ verbose(env, "BPF_MOV uses reserved fields\n"); + return -EINVAL; + } + + /* check src operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + } else { + if (insn->src_reg != BPF_REG_0 || insn->off != 0) { +- verbose("BPF_MOV uses reserved fields\n"); ++ verbose(env, "BPF_MOV uses reserved fields\n"); + return -EINVAL; + } + } + +- /* check dest operand */ +- err = check_reg_arg(regs, insn->dst_reg, DST_OP); ++ /* check dest operand, mark as required later */ ++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); + if (err) + return err; + + if (BPF_SRC(insn->code) == BPF_X) { ++ struct bpf_reg_state *src_reg = regs + insn->src_reg; ++ struct bpf_reg_state *dst_reg = regs + insn->dst_reg; ++ + if (BPF_CLASS(insn->code) == BPF_ALU64) { + /* case: R1 = R2 + * copy register state to dest reg + */ +- regs[insn->dst_reg] = regs[insn->src_reg]; ++ *dst_reg = *src_reg; ++ dst_reg->live |= REG_LIVE_WRITTEN; ++ dst_reg->subreg_def = DEF_NOT_SUBREG; + } else { ++ /* R1 = (u32) R2 */ + if (is_pointer_value(env, insn->src_reg)) { +- verbose("R%d partial copy of pointer\n", ++ verbose(env, ++ "R%d partial copy of pointer\n", + insn->src_reg); + return -EACCES; ++ } else if (src_reg->type == SCALAR_VALUE) { ++ *dst_reg = *src_reg; ++ dst_reg->live |= REG_LIVE_WRITTEN; ++ dst_reg->subreg_def = env->insn_idx + 1; ++ } else { ++ mark_reg_unknown(env, regs, ++ insn->dst_reg); + } +- regs[insn->dst_reg].type = UNKNOWN_VALUE; +- regs[insn->dst_reg].map_ptr = NULL; ++ coerce_reg_to_size(dst_reg, 4); + } + } else { + /* case: R = imm + * remember the value we stored into this reg + */ +- regs[insn->dst_reg].type = CONST_IMM; +- regs[insn->dst_reg].imm = insn->imm; ++ /* clear any state __mark_reg_known doesn't set */ ++ mark_reg_unknown(env, regs, insn->dst_reg); ++ regs[insn->dst_reg].type = SCALAR_VALUE; ++ if (BPF_CLASS(insn->code) == BPF_ALU64) { ++ __mark_reg_known(regs + insn->dst_reg, ++ insn->imm); ++ } else { ++ __mark_reg_known(regs + insn->dst_reg, ++ (u32)insn->imm); ++ } + } + + } else if (opcode > BPF_END) { +- verbose("invalid BPF_ALU opcode %x\n", opcode); ++ verbose(env, "invalid BPF_ALU opcode %x\n", opcode); + return -EINVAL; + + } else { /* all other ALU ops: and, sub, xor, add, ... */ + +- bool stack_relative = false; +- + if (BPF_SRC(insn->code) == BPF_X) { + if (insn->imm != 0 || insn->off != 0) { +- verbose("BPF_ALU uses reserved fields\n"); ++ verbose(env, "BPF_ALU uses reserved fields\n"); + return -EINVAL; + } + /* check src1 operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + } else { + if (insn->src_reg != BPF_REG_0 || insn->off != 0) { +- verbose("BPF_ALU uses reserved fields\n"); ++ verbose(env, "BPF_ALU uses reserved fields\n"); + return -EINVAL; + } + } + + /* check src2 operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + if ((opcode == BPF_MOD || opcode == BPF_DIV) && + BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { +- verbose("div by zero\n"); ++ verbose(env, "div by zero\n"); + return -EINVAL; + } + +@@ -1148,185 +5268,980 @@ static int check_alu_op(struct verifier_ + int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; + + if (insn->imm < 0 || insn->imm >= size) { +- verbose("invalid shift %d\n", insn->imm); ++ verbose(env, "invalid shift %d\n", insn->imm); + return -EINVAL; + } + } + +- /* pattern match 'bpf_add Rx, imm' instruction */ +- if (opcode == BPF_ADD && BPF_CLASS(insn->code) == BPF_ALU64 && +- regs[insn->dst_reg].type == FRAME_PTR && +- BPF_SRC(insn->code) == BPF_K) { +- stack_relative = true; +- } else if (is_pointer_value(env, insn->dst_reg)) { +- verbose("R%d pointer arithmetic prohibited\n", +- insn->dst_reg); +- return -EACCES; +- } else if (BPF_SRC(insn->code) == BPF_X && +- is_pointer_value(env, insn->src_reg)) { +- verbose("R%d pointer arithmetic prohibited\n", +- insn->src_reg); +- return -EACCES; +- } +- + /* check dest operand */ +- err = check_reg_arg(regs, insn->dst_reg, DST_OP); ++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); + if (err) + return err; + +- if (stack_relative) { +- regs[insn->dst_reg].type = PTR_TO_STACK; +- regs[insn->dst_reg].imm = insn->imm; +- } ++ return adjust_reg_min_max_vals(env, insn); + } + + return 0; + } + +-static int check_cond_jmp_op(struct verifier_env *env, ++static void __find_good_pkt_pointers(struct bpf_func_state *state, ++ struct bpf_reg_state *dst_reg, ++ enum bpf_reg_type type, u16 new_range) ++{ ++ struct bpf_reg_state *reg; ++ int i; ++ ++ for (i = 0; i < MAX_BPF_REG; i++) { ++ reg = &state->regs[i]; ++ if (reg->type == type && reg->id == dst_reg->id) ++ /* keep the maximum range already checked */ ++ reg->range = max(reg->range, new_range); ++ } ++ ++ bpf_for_each_spilled_reg(i, state, reg) { ++ if (!reg) ++ continue; ++ if (reg->type == type && reg->id == dst_reg->id) ++ reg->range = max(reg->range, new_range); ++ } ++} ++ ++static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, ++ struct bpf_reg_state *dst_reg, ++ enum bpf_reg_type type, ++ bool range_right_open) ++{ ++ u16 new_range; ++ int i; ++ ++ if (dst_reg->off < 0 || ++ (dst_reg->off == 0 && range_right_open)) ++ /* This doesn't give us any range */ ++ return; ++ ++ if (dst_reg->umax_value > MAX_PACKET_OFF || ++ dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) ++ /* Risk of overflow. For instance, ptr + (1<<63) may be less ++ * than pkt_end, but that's because it's also less than pkt. ++ */ ++ return; ++ ++ new_range = dst_reg->off; ++ if (range_right_open) ++ new_range--; ++ ++ /* Examples for register markings: ++ * ++ * pkt_data in dst register: ++ * ++ * r2 = r3; ++ * r2 += 8; ++ * if (r2 > pkt_end) goto ++ * ++ * ++ * r2 = r3; ++ * r2 += 8; ++ * if (r2 < pkt_end) goto ++ * ++ * ++ * Where: ++ * r2 == dst_reg, pkt_end == src_reg ++ * r2=pkt(id=n,off=8,r=0) ++ * r3=pkt(id=n,off=0,r=0) ++ * ++ * pkt_data in src register: ++ * ++ * r2 = r3; ++ * r2 += 8; ++ * if (pkt_end >= r2) goto ++ * ++ * ++ * r2 = r3; ++ * r2 += 8; ++ * if (pkt_end <= r2) goto ++ * ++ * ++ * Where: ++ * pkt_end == dst_reg, r2 == src_reg ++ * r2=pkt(id=n,off=8,r=0) ++ * r3=pkt(id=n,off=0,r=0) ++ * ++ * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) ++ * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) ++ * and [r3, r3 + 8-1) respectively is safe to access depending on ++ * the check. ++ */ ++ ++ /* If our ids match, then we must have the same max_value. And we ++ * don't care about the other reg's fixed offset, since if it's too big ++ * the range won't allow anything. ++ * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. ++ */ ++ for (i = 0; i <= vstate->curframe; i++) ++ __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, ++ new_range); ++} ++ ++/* compute branch direction of the expression "if (reg opcode val) goto target;" ++ * and return: ++ * 1 - branch will be taken and "goto target" will be executed ++ * 0 - branch will not be taken and fall-through to next insn ++ * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10] ++ */ ++static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, ++ bool is_jmp32) ++{ ++ struct bpf_reg_state reg_lo; ++ s64 sval; ++ ++ if (__is_pointer_value(false, reg)) ++ return -1; ++ ++ if (is_jmp32) { ++ reg_lo = *reg; ++ reg = ®_lo; ++ /* For JMP32, only low 32 bits are compared, coerce_reg_to_size ++ * could truncate high bits and update umin/umax according to ++ * information of low bits. ++ */ ++ coerce_reg_to_size(reg, 4); ++ /* smin/smax need special handling. For example, after coerce, ++ * if smin_value is 0x00000000ffffffffLL, the value is -1 when ++ * used as operand to JMP32. It is a negative number from s32's ++ * point of view, while it is a positive number when seen as ++ * s64. The smin/smax are kept as s64, therefore, when used with ++ * JMP32, they need to be transformed into s32, then sign ++ * extended back to s64. ++ * ++ * Also, smin/smax were copied from umin/umax. If umin/umax has ++ * different sign bit, then min/max relationship doesn't ++ * maintain after casting into s32, for this case, set smin/smax ++ * to safest range. ++ */ ++ if ((reg->umax_value ^ reg->umin_value) & ++ (1ULL << 31)) { ++ reg->smin_value = S32_MIN; ++ reg->smax_value = S32_MAX; ++ } ++ reg->smin_value = (s64)(s32)reg->smin_value; ++ reg->smax_value = (s64)(s32)reg->smax_value; ++ ++ val = (u32)val; ++ sval = (s64)(s32)val; ++ } else { ++ sval = (s64)val; ++ } ++ ++ switch (opcode) { ++ case BPF_JEQ: ++ if (tnum_is_const(reg->var_off)) ++ return !!tnum_equals_const(reg->var_off, val); ++ break; ++ case BPF_JNE: ++ if (tnum_is_const(reg->var_off)) ++ return !tnum_equals_const(reg->var_off, val); ++ break; ++ case BPF_JSET: ++ if ((~reg->var_off.mask & reg->var_off.value) & val) ++ return 1; ++ if (!((reg->var_off.mask | reg->var_off.value) & val)) ++ return 0; ++ break; ++ case BPF_JGT: ++ if (reg->umin_value > val) ++ return 1; ++ else if (reg->umax_value <= val) ++ return 0; ++ break; ++ case BPF_JSGT: ++ if (reg->smin_value > sval) ++ return 1; ++ else if (reg->smax_value < sval) ++ return 0; ++ break; ++ case BPF_JLT: ++ if (reg->umax_value < val) ++ return 1; ++ else if (reg->umin_value >= val) ++ return 0; ++ break; ++ case BPF_JSLT: ++ if (reg->smax_value < sval) ++ return 1; ++ else if (reg->smin_value >= sval) ++ return 0; ++ break; ++ case BPF_JGE: ++ if (reg->umin_value >= val) ++ return 1; ++ else if (reg->umax_value < val) ++ return 0; ++ break; ++ case BPF_JSGE: ++ if (reg->smin_value >= sval) ++ return 1; ++ else if (reg->smax_value < sval) ++ return 0; ++ break; ++ case BPF_JLE: ++ if (reg->umax_value <= val) ++ return 1; ++ else if (reg->umin_value > val) ++ return 0; ++ break; ++ case BPF_JSLE: ++ if (reg->smax_value <= sval) ++ return 1; ++ else if (reg->smin_value > sval) ++ return 0; ++ break; ++ } ++ ++ return -1; ++} ++ ++/* Generate min value of the high 32-bit from TNUM info. */ ++static u64 gen_hi_min(struct tnum var) ++{ ++ return var.value & ~0xffffffffULL; ++} ++ ++/* Generate max value of the high 32-bit from TNUM info. */ ++static u64 gen_hi_max(struct tnum var) ++{ ++ return (var.value | var.mask) & ~0xffffffffULL; ++} ++ ++/* Return true if VAL is compared with a s64 sign extended from s32, and they ++ * are with the same signedness. ++ */ ++static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg) ++{ ++ return ((s32)sval >= 0 && ++ reg->smin_value >= 0 && reg->smax_value <= S32_MAX) || ++ ((s32)sval < 0 && ++ reg->smax_value <= 0 && reg->smin_value >= S32_MIN); ++} ++ ++/* Constrain the possible values of @reg with unsigned upper bound @bound. ++ * If @is_exclusive, @bound is an exclusive limit, otherwise it is inclusive. ++ * If @is_jmp32, @bound is a 32-bit value that only constrains the low 32 bits ++ * of @reg. ++ */ ++static void set_upper_bound(struct bpf_reg_state *reg, u64 bound, bool is_jmp32, ++ bool is_exclusive) ++{ ++ if (is_exclusive) { ++ /* There are no values for `reg` that make `reg<0` true. */ ++ if (bound == 0) ++ return; ++ bound--; ++ } ++ if (is_jmp32) { ++ /* Constrain the register's value in the tnum representation. ++ * For 64-bit comparisons this happens later in ++ * __reg_bound_offset(), but for 32-bit comparisons, we can be ++ * more precise than what can be derived from the updated ++ * numeric bounds. ++ */ ++ struct tnum t = tnum_range(0, bound); ++ ++ t.mask |= ~0xffffffffULL; /* upper half is unknown */ ++ reg->var_off = tnum_intersect(reg->var_off, t); ++ ++ /* Compute the 64-bit bound from the 32-bit bound. */ ++ bound += gen_hi_max(reg->var_off); ++ } ++ reg->umax_value = min(reg->umax_value, bound); ++} ++ ++/* Constrain the possible values of @reg with unsigned lower bound @bound. ++ * If @is_exclusive, @bound is an exclusive limit, otherwise it is inclusive. ++ * If @is_jmp32, @bound is a 32-bit value that only constrains the low 32 bits ++ * of @reg. ++ */ ++static void set_lower_bound(struct bpf_reg_state *reg, u64 bound, bool is_jmp32, ++ bool is_exclusive) ++{ ++ if (is_exclusive) { ++ /* There are no values for `reg` that make `reg>MAX` true. */ ++ if (bound == (is_jmp32 ? U32_MAX : U64_MAX)) ++ return; ++ bound++; ++ } ++ if (is_jmp32) { ++ /* Constrain the register's value in the tnum representation. ++ * For 64-bit comparisons this happens later in ++ * __reg_bound_offset(), but for 32-bit comparisons, we can be ++ * more precise than what can be derived from the updated ++ * numeric bounds. ++ */ ++ struct tnum t = tnum_range(bound, U32_MAX); ++ ++ t.mask |= ~0xffffffffULL; /* upper half is unknown */ ++ reg->var_off = tnum_intersect(reg->var_off, t); ++ ++ /* Compute the 64-bit bound from the 32-bit bound. */ ++ bound += gen_hi_min(reg->var_off); ++ } ++ reg->umin_value = max(reg->umin_value, bound); ++} ++ ++/* Adjusts the register min/max values in the case that the dst_reg is the ++ * variable register that we are working on, and src_reg is a constant or we're ++ * simply doing a BPF_K check. ++ * In JEQ/JNE cases we also adjust the var_off values. ++ */ ++static void reg_set_min_max(struct bpf_reg_state *true_reg, ++ struct bpf_reg_state *false_reg, u64 val, ++ u8 opcode, bool is_jmp32) ++{ ++ s64 sval; ++ ++ /* If the dst_reg is a pointer, we can't learn anything about its ++ * variable offset from the compare (unless src_reg were a pointer into ++ * the same object, but we don't bother with that. ++ * Since false_reg and true_reg have the same type by construction, we ++ * only need to check one of them for pointerness. ++ */ ++ if (__is_pointer_value(false, false_reg)) ++ return; ++ ++ val = is_jmp32 ? (u32)val : val; ++ sval = is_jmp32 ? (s64)(s32)val : (s64)val; ++ ++ switch (opcode) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ { ++ struct bpf_reg_state *reg = ++ opcode == BPF_JEQ ? true_reg : false_reg; ++ ++ /* For BPF_JEQ, if this is false we know nothing Jon Snow, but ++ * if it is true we know the value for sure. Likewise for ++ * BPF_JNE. ++ */ ++ if (is_jmp32) { ++ u64 old_v = reg->var_off.value; ++ u64 hi_mask = ~0xffffffffULL; ++ ++ reg->var_off.value = (old_v & hi_mask) | val; ++ reg->var_off.mask &= hi_mask; ++ } else { ++ __mark_reg_known(reg, val); ++ } ++ break; ++ } ++ case BPF_JSET: ++ false_reg->var_off = tnum_and(false_reg->var_off, ++ tnum_const(~val)); ++ if (is_power_of_2(val)) ++ true_reg->var_off = tnum_or(true_reg->var_off, ++ tnum_const(val)); ++ break; ++ case BPF_JGE: ++ case BPF_JGT: ++ { ++ set_upper_bound(false_reg, val, is_jmp32, opcode == BPF_JGE); ++ set_lower_bound(true_reg, val, is_jmp32, opcode == BPF_JGT); ++ break; ++ } ++ case BPF_JSGE: ++ case BPF_JSGT: ++ { ++ s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; ++ s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; ++ ++ /* If the full s64 was not sign-extended from s32 then don't ++ * deduct further info. ++ */ ++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) ++ break; ++ false_reg->smax_value = min(false_reg->smax_value, false_smax); ++ true_reg->smin_value = max(true_reg->smin_value, true_smin); ++ break; ++ } ++ case BPF_JLE: ++ case BPF_JLT: ++ { ++ set_lower_bound(false_reg, val, is_jmp32, opcode == BPF_JLE); ++ set_upper_bound(true_reg, val, is_jmp32, opcode == BPF_JLT); ++ break; ++ } ++ case BPF_JSLE: ++ case BPF_JSLT: ++ { ++ s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; ++ s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; ++ ++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) ++ break; ++ false_reg->smin_value = max(false_reg->smin_value, false_smin); ++ true_reg->smax_value = min(true_reg->smax_value, true_smax); ++ break; ++ } ++ default: ++ break; ++ } ++ ++ __reg_deduce_bounds(false_reg); ++ __reg_deduce_bounds(true_reg); ++ /* We might have learned some bits from the bounds. */ ++ __reg_bound_offset(false_reg); ++ __reg_bound_offset(true_reg); ++ /* Intersecting with the old var_off might have improved our bounds ++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), ++ * then new var_off is (0; 0x7f...fc) which improves our umax. ++ */ ++ __update_reg_bounds(false_reg); ++ __update_reg_bounds(true_reg); ++} ++ ++/* Same as above, but for the case that dst_reg holds a constant and src_reg is ++ * the variable reg. ++ */ ++static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, ++ struct bpf_reg_state *false_reg, u64 val, ++ u8 opcode, bool is_jmp32) ++{ ++ s64 sval; ++ ++ if (__is_pointer_value(false, false_reg)) ++ return; ++ ++ val = is_jmp32 ? (u32)val : val; ++ sval = is_jmp32 ? (s64)(s32)val : (s64)val; ++ ++ switch (opcode) { ++ case BPF_JEQ: ++ case BPF_JNE: ++ { ++ struct bpf_reg_state *reg = ++ opcode == BPF_JEQ ? true_reg : false_reg; ++ ++ if (is_jmp32) { ++ u64 old_v = reg->var_off.value; ++ u64 hi_mask = ~0xffffffffULL; ++ ++ reg->var_off.value = (old_v & hi_mask) | val; ++ reg->var_off.mask &= hi_mask; ++ } else { ++ __mark_reg_known(reg, val); ++ } ++ break; ++ } ++ case BPF_JSET: ++ false_reg->var_off = tnum_and(false_reg->var_off, ++ tnum_const(~val)); ++ if (is_power_of_2(val)) ++ true_reg->var_off = tnum_or(true_reg->var_off, ++ tnum_const(val)); ++ break; ++ case BPF_JGE: ++ case BPF_JGT: ++ { ++ set_lower_bound(false_reg, val, is_jmp32, opcode == BPF_JGE); ++ set_upper_bound(true_reg, val, is_jmp32, opcode == BPF_JGT); ++ break; ++ } ++ case BPF_JSGE: ++ case BPF_JSGT: ++ { ++ s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1; ++ s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval; ++ ++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) ++ break; ++ false_reg->smin_value = max(false_reg->smin_value, false_smin); ++ true_reg->smax_value = min(true_reg->smax_value, true_smax); ++ break; ++ } ++ case BPF_JLE: ++ case BPF_JLT: ++ { ++ set_upper_bound(false_reg, val, is_jmp32, opcode == BPF_JLE); ++ set_lower_bound(true_reg, val, is_jmp32, opcode == BPF_JLT); ++ break; ++ } ++ case BPF_JSLE: ++ case BPF_JSLT: ++ { ++ s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1; ++ s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval; ++ ++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) ++ break; ++ false_reg->smax_value = min(false_reg->smax_value, false_smax); ++ true_reg->smin_value = max(true_reg->smin_value, true_smin); ++ break; ++ } ++ default: ++ break; ++ } ++ ++ __reg_deduce_bounds(false_reg); ++ __reg_deduce_bounds(true_reg); ++ /* We might have learned some bits from the bounds. */ ++ __reg_bound_offset(false_reg); ++ __reg_bound_offset(true_reg); ++ /* Intersecting with the old var_off might have improved our bounds ++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), ++ * then new var_off is (0; 0x7f...fc) which improves our umax. ++ */ ++ __update_reg_bounds(false_reg); ++ __update_reg_bounds(true_reg); ++} ++ ++/* Regs are known to be equal, so intersect their min/max/var_off */ ++static void __reg_combine_min_max(struct bpf_reg_state *src_reg, ++ struct bpf_reg_state *dst_reg) ++{ ++ src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, ++ dst_reg->umin_value); ++ src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, ++ dst_reg->umax_value); ++ src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, ++ dst_reg->smin_value); ++ src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, ++ dst_reg->smax_value); ++ src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, ++ dst_reg->var_off); ++ /* We might have learned new bounds from the var_off. */ ++ __update_reg_bounds(src_reg); ++ __update_reg_bounds(dst_reg); ++ /* We might have learned something about the sign bit. */ ++ __reg_deduce_bounds(src_reg); ++ __reg_deduce_bounds(dst_reg); ++ /* We might have learned some bits from the bounds. */ ++ __reg_bound_offset(src_reg); ++ __reg_bound_offset(dst_reg); ++ /* Intersecting with the old var_off might have improved our bounds ++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), ++ * then new var_off is (0; 0x7f...fc) which improves our umax. ++ */ ++ __update_reg_bounds(src_reg); ++ __update_reg_bounds(dst_reg); ++} ++ ++static void reg_combine_min_max(struct bpf_reg_state *true_src, ++ struct bpf_reg_state *true_dst, ++ struct bpf_reg_state *false_src, ++ struct bpf_reg_state *false_dst, ++ u8 opcode) ++{ ++ switch (opcode) { ++ case BPF_JEQ: ++ __reg_combine_min_max(true_src, true_dst); ++ break; ++ case BPF_JNE: ++ __reg_combine_min_max(false_src, false_dst); ++ break; ++ } ++} ++ ++static void mark_ptr_or_null_reg(struct bpf_func_state *state, ++ struct bpf_reg_state *reg, u32 id, ++ bool is_null) ++{ ++ if (reg_type_may_be_null(reg->type) && reg->id == id) { ++ /* Old offset (both fixed and variable parts) should ++ * have been known-zero, because we don't allow pointer ++ * arithmetic on pointers that might be NULL. ++ */ ++ if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || ++ !tnum_equals_const(reg->var_off, 0) || ++ reg->off)) { ++ __mark_reg_known_zero(reg); ++ reg->off = 0; ++ } ++ if (is_null) { ++ reg->type = SCALAR_VALUE; ++ } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) { ++ if (reg->map_ptr->inner_map_meta) { ++ reg->type = CONST_PTR_TO_MAP; ++ reg->map_ptr = reg->map_ptr->inner_map_meta; ++ } else if (reg->map_ptr->map_type == ++ BPF_MAP_TYPE_XSKMAP) { ++ reg->type = PTR_TO_XDP_SOCK; ++ } else { ++ reg->type = PTR_TO_MAP_VALUE; ++ } ++ } else if (reg->type == PTR_TO_SOCKET_OR_NULL) { ++ reg->type = PTR_TO_SOCKET; ++ } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) { ++ reg->type = PTR_TO_SOCK_COMMON; ++ } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { ++ reg->type = PTR_TO_TCP_SOCK; ++ } ++ if (is_null) { ++ /* We don't need id and ref_obj_id from this point ++ * onwards anymore, thus we should better reset it, ++ * so that state pruning has chances to take effect. ++ */ ++ reg->id = 0; ++ reg->ref_obj_id = 0; ++ } else if (!reg_may_point_to_spin_lock(reg)) { ++ /* For not-NULL ptr, reg->ref_obj_id will be reset ++ * in release_reg_references(). ++ * ++ * reg->id is still used by spin_lock ptr. Other ++ * than spin_lock ptr type, reg->id can be reset. ++ */ ++ reg->id = 0; ++ } ++ } ++} ++ ++static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, ++ bool is_null) ++{ ++ struct bpf_reg_state *reg; ++ int i; ++ ++ for (i = 0; i < MAX_BPF_REG; i++) ++ mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); ++ ++ bpf_for_each_spilled_reg(i, state, reg) { ++ if (!reg) ++ continue; ++ mark_ptr_or_null_reg(state, reg, id, is_null); ++ } ++} ++ ++/* The logic is similar to find_good_pkt_pointers(), both could eventually ++ * be folded together at some point. ++ */ ++static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, ++ bool is_null) ++{ ++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; ++ struct bpf_reg_state *regs = state->regs; ++ u32 ref_obj_id = regs[regno].ref_obj_id; ++ u32 id = regs[regno].id; ++ int i; ++ ++ if (ref_obj_id && ref_obj_id == id && is_null) ++ /* regs[regno] is in the " == NULL" branch. ++ * No one could have freed the reference state before ++ * doing the NULL check. ++ */ ++ WARN_ON_ONCE(release_reference_state(state, id)); ++ ++ for (i = 0; i <= vstate->curframe; i++) ++ __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); ++} ++ ++static bool try_match_pkt_pointers(const struct bpf_insn *insn, ++ struct bpf_reg_state *dst_reg, ++ struct bpf_reg_state *src_reg, ++ struct bpf_verifier_state *this_branch, ++ struct bpf_verifier_state *other_branch) ++{ ++ if (BPF_SRC(insn->code) != BPF_X) ++ return false; ++ ++ /* Pointers are always 64-bit. */ ++ if (BPF_CLASS(insn->code) == BPF_JMP32) ++ return false; ++ ++ switch (BPF_OP(insn->code)) { ++ case BPF_JGT: ++ if ((dst_reg->type == PTR_TO_PACKET && ++ src_reg->type == PTR_TO_PACKET_END) || ++ (dst_reg->type == PTR_TO_PACKET_META && ++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { ++ /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ ++ find_good_pkt_pointers(this_branch, dst_reg, ++ dst_reg->type, false); ++ } else if ((dst_reg->type == PTR_TO_PACKET_END && ++ src_reg->type == PTR_TO_PACKET) || ++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && ++ src_reg->type == PTR_TO_PACKET_META)) { ++ /* pkt_end > pkt_data', pkt_data > pkt_meta' */ ++ find_good_pkt_pointers(other_branch, src_reg, ++ src_reg->type, true); ++ } else { ++ return false; ++ } ++ break; ++ case BPF_JLT: ++ if ((dst_reg->type == PTR_TO_PACKET && ++ src_reg->type == PTR_TO_PACKET_END) || ++ (dst_reg->type == PTR_TO_PACKET_META && ++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { ++ /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ ++ find_good_pkt_pointers(other_branch, dst_reg, ++ dst_reg->type, true); ++ } else if ((dst_reg->type == PTR_TO_PACKET_END && ++ src_reg->type == PTR_TO_PACKET) || ++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && ++ src_reg->type == PTR_TO_PACKET_META)) { ++ /* pkt_end < pkt_data', pkt_data > pkt_meta' */ ++ find_good_pkt_pointers(this_branch, src_reg, ++ src_reg->type, false); ++ } else { ++ return false; ++ } ++ break; ++ case BPF_JGE: ++ if ((dst_reg->type == PTR_TO_PACKET && ++ src_reg->type == PTR_TO_PACKET_END) || ++ (dst_reg->type == PTR_TO_PACKET_META && ++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { ++ /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ ++ find_good_pkt_pointers(this_branch, dst_reg, ++ dst_reg->type, true); ++ } else if ((dst_reg->type == PTR_TO_PACKET_END && ++ src_reg->type == PTR_TO_PACKET) || ++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && ++ src_reg->type == PTR_TO_PACKET_META)) { ++ /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ ++ find_good_pkt_pointers(other_branch, src_reg, ++ src_reg->type, false); ++ } else { ++ return false; ++ } ++ break; ++ case BPF_JLE: ++ if ((dst_reg->type == PTR_TO_PACKET && ++ src_reg->type == PTR_TO_PACKET_END) || ++ (dst_reg->type == PTR_TO_PACKET_META && ++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { ++ /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ ++ find_good_pkt_pointers(other_branch, dst_reg, ++ dst_reg->type, false); ++ } else if ((dst_reg->type == PTR_TO_PACKET_END && ++ src_reg->type == PTR_TO_PACKET) || ++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && ++ src_reg->type == PTR_TO_PACKET_META)) { ++ /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ ++ find_good_pkt_pointers(this_branch, src_reg, ++ src_reg->type, true); ++ } else { ++ return false; ++ } ++ break; ++ default: ++ return false; ++ } ++ ++ return true; ++} ++ ++static int check_cond_jmp_op(struct bpf_verifier_env *env, + struct bpf_insn *insn, int *insn_idx) + { +- struct reg_state *regs = env->cur_state.regs; +- struct verifier_state *other_branch; ++ struct bpf_verifier_state *this_branch = env->cur_state; ++ struct bpf_verifier_state *other_branch; ++ struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; ++ struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; + u8 opcode = BPF_OP(insn->code); ++ bool is_jmp32; ++ int pred = -1; + int err; + +- if (opcode > BPF_EXIT) { +- verbose("invalid BPF_JMP opcode %x\n", opcode); ++ /* Only conditional jumps are expected to reach here. */ ++ if (opcode == BPF_JA || opcode > BPF_JSLE) { ++ verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); + return -EINVAL; + } + + if (BPF_SRC(insn->code) == BPF_X) { + if (insn->imm != 0) { +- verbose("BPF_JMP uses reserved fields\n"); ++ verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); + return -EINVAL; + } + + /* check src1 operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + + if (is_pointer_value(env, insn->src_reg)) { +- verbose("R%d pointer comparison prohibited\n", ++ verbose(env, "R%d pointer comparison prohibited\n", + insn->src_reg); + return -EACCES; + } ++ src_reg = ®s[insn->src_reg]; + } else { + if (insn->src_reg != BPF_REG_0) { +- verbose("BPF_JMP uses reserved fields\n"); ++ verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); + return -EINVAL; + } + } + + /* check src2 operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + +- /* detect if R == 0 where R was initialized to zero earlier */ +- if (BPF_SRC(insn->code) == BPF_K && +- (opcode == BPF_JEQ || opcode == BPF_JNE) && +- regs[insn->dst_reg].type == CONST_IMM && +- regs[insn->dst_reg].imm == insn->imm) { +- if (opcode == BPF_JEQ) { +- /* if (imm == imm) goto pc+off; +- * only follow the goto, ignore fall-through +- */ +- *insn_idx += insn->off; +- return 0; +- } else { +- /* if (imm != imm) goto pc+off; +- * only follow fall-through branch, since +- * that's where the program will go +- */ +- return 0; +- } ++ dst_reg = ®s[insn->dst_reg]; ++ is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; ++ ++ if (BPF_SRC(insn->code) == BPF_K) ++ pred = is_branch_taken(dst_reg, insn->imm, ++ opcode, is_jmp32); ++ else if (src_reg->type == SCALAR_VALUE && ++ tnum_is_const(src_reg->var_off)) ++ pred = is_branch_taken(dst_reg, src_reg->var_off.value, ++ opcode, is_jmp32); ++ if (pred >= 0) { ++ err = mark_chain_precision(env, insn->dst_reg); ++ if (BPF_SRC(insn->code) == BPF_X && !err) ++ err = mark_chain_precision(env, insn->src_reg); ++ if (err) ++ return err; ++ } ++ ++ if (pred == 1) { ++ /* Only follow the goto, ignore fall-through. If needed, push ++ * the fall-through branch for simulation under speculative ++ * execution. ++ */ ++ if (!env->allow_ptr_leaks && ++ !sanitize_speculative_path(env, insn, *insn_idx + 1, ++ *insn_idx)) ++ return -EFAULT; ++ *insn_idx += insn->off; ++ return 0; ++ } else if (pred == 0) { ++ /* Only follow the fall-through branch, since that's where the ++ * program will go. If needed, push the goto branch for ++ * simulation under speculative execution. ++ */ ++ if (!env->allow_ptr_leaks && ++ !sanitize_speculative_path(env, insn, ++ *insn_idx + insn->off + 1, ++ *insn_idx)) ++ return -EFAULT; ++ return 0; + } + +- other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx); ++ other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, ++ false); + if (!other_branch) + return -EFAULT; ++ other_branch_regs = other_branch->frame[other_branch->curframe]->regs; + +- /* detect if R == 0 where R is returned value from bpf_map_lookup_elem() */ +- if (BPF_SRC(insn->code) == BPF_K && +- insn->imm == 0 && (opcode == BPF_JEQ || +- opcode == BPF_JNE) && +- regs[insn->dst_reg].type == PTR_TO_MAP_VALUE_OR_NULL) { +- if (opcode == BPF_JEQ) { +- /* next fallthrough insn can access memory via +- * this register +- */ +- regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; +- /* branch targer cannot access it, since reg == 0 */ +- other_branch->regs[insn->dst_reg].type = CONST_IMM; +- other_branch->regs[insn->dst_reg].imm = 0; +- } else { +- other_branch->regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; +- regs[insn->dst_reg].type = CONST_IMM; +- regs[insn->dst_reg].imm = 0; +- } +- } else if (is_pointer_value(env, insn->dst_reg)) { +- verbose("R%d pointer comparison prohibited\n", insn->dst_reg); ++ /* detect if we are comparing against a constant value so we can adjust ++ * our min/max values for our dst register. ++ * this is only legit if both are scalars (or pointers to the same ++ * object, I suppose, but we don't support that right now), because ++ * otherwise the different base pointers mean the offsets aren't ++ * comparable. ++ */ ++ if (BPF_SRC(insn->code) == BPF_X) { ++ struct bpf_reg_state *src_reg = ®s[insn->src_reg]; ++ struct bpf_reg_state lo_reg0 = *dst_reg; ++ struct bpf_reg_state lo_reg1 = *src_reg; ++ struct bpf_reg_state *src_lo, *dst_lo; ++ ++ dst_lo = &lo_reg0; ++ src_lo = &lo_reg1; ++ coerce_reg_to_size(dst_lo, 4); ++ coerce_reg_to_size(src_lo, 4); ++ ++ if (dst_reg->type == SCALAR_VALUE && ++ src_reg->type == SCALAR_VALUE) { ++ if (tnum_is_const(src_reg->var_off) || ++ (is_jmp32 && tnum_is_const(src_lo->var_off))) ++ reg_set_min_max(&other_branch_regs[insn->dst_reg], ++ dst_reg, ++ is_jmp32 ++ ? src_lo->var_off.value ++ : src_reg->var_off.value, ++ opcode, is_jmp32); ++ else if (tnum_is_const(dst_reg->var_off) || ++ (is_jmp32 && tnum_is_const(dst_lo->var_off))) ++ reg_set_min_max_inv(&other_branch_regs[insn->src_reg], ++ src_reg, ++ is_jmp32 ++ ? dst_lo->var_off.value ++ : dst_reg->var_off.value, ++ opcode, is_jmp32); ++ else if (!is_jmp32 && ++ (opcode == BPF_JEQ || opcode == BPF_JNE)) ++ /* Comparing for equality, we can combine knowledge */ ++ reg_combine_min_max(&other_branch_regs[insn->src_reg], ++ &other_branch_regs[insn->dst_reg], ++ src_reg, dst_reg, opcode); ++ } ++ } else if (dst_reg->type == SCALAR_VALUE) { ++ reg_set_min_max(&other_branch_regs[insn->dst_reg], ++ dst_reg, insn->imm, opcode, is_jmp32); ++ } ++ ++ /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). ++ * NOTE: these optimizations below are related with pointer comparison ++ * which will never be JMP32. ++ */ ++ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && ++ insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && ++ reg_type_may_be_null(dst_reg->type)) { ++ /* Mark all identical registers in each branch as either ++ * safe or unknown depending R == 0 or R != 0 conditional. ++ */ ++ mark_ptr_or_null_regs(this_branch, insn->dst_reg, ++ opcode == BPF_JNE); ++ mark_ptr_or_null_regs(other_branch, insn->dst_reg, ++ opcode == BPF_JEQ); ++ } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], ++ this_branch, other_branch) && ++ is_pointer_value(env, insn->dst_reg)) { ++ verbose(env, "R%d pointer comparison prohibited\n", ++ insn->dst_reg); + return -EACCES; +- } else if (BPF_SRC(insn->code) == BPF_K && +- (opcode == BPF_JEQ || opcode == BPF_JNE)) { +- +- if (opcode == BPF_JEQ) { +- /* detect if (R == imm) goto +- * and in the target state recognize that R = imm +- */ +- other_branch->regs[insn->dst_reg].type = CONST_IMM; +- other_branch->regs[insn->dst_reg].imm = insn->imm; +- } else { +- /* detect if (R != imm) goto +- * and in the fall-through state recognize that R = imm +- */ +- regs[insn->dst_reg].type = CONST_IMM; +- regs[insn->dst_reg].imm = insn->imm; +- } + } +- if (log_level) +- print_verifier_state(env); ++ if (env->log.level & BPF_LOG_LEVEL) ++ print_verifier_state(env, this_branch->frame[this_branch->curframe]); + return 0; + } + +-/* return the map pointer stored inside BPF_LD_IMM64 instruction */ +-static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn) +-{ +- u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32; +- +- return (struct bpf_map *) (unsigned long) imm64; +-} +- + /* verify BPF_LD_IMM64 instruction */ +-static int check_ld_imm(struct verifier_env *env, struct bpf_insn *insn) ++static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) + { +- struct reg_state *regs = env->cur_state.regs; ++ struct bpf_insn_aux_data *aux = cur_aux(env); ++ struct bpf_reg_state *regs = cur_regs(env); ++ struct bpf_map *map; + int err; + + if (BPF_SIZE(insn->code) != BPF_DW) { +- verbose("invalid BPF_LD_IMM insn\n"); ++ verbose(env, "invalid BPF_LD_IMM insn\n"); + return -EINVAL; + } + if (insn->off != 0) { +- verbose("BPF_LD_IMM64 uses reserved fields\n"); ++ verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); + return -EINVAL; + } + +- err = check_reg_arg(regs, insn->dst_reg, DST_OP); ++ err = check_reg_arg(env, insn->dst_reg, DST_OP); + if (err) + return err; + +- if (insn->src_reg == 0) +- /* generic move 64-bit immediate into a register */ ++ if (insn->src_reg == 0) { ++ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; ++ ++ regs[insn->dst_reg].type = SCALAR_VALUE; ++ __mark_reg_known(®s[insn->dst_reg], imm); + return 0; ++ } + +- /* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */ +- BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD); ++ map = env->used_maps[aux->map_index]; ++ mark_reg_known_zero(env, regs, insn->dst_reg); ++ regs[insn->dst_reg].map_ptr = map; ++ ++ if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { ++ regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; ++ regs[insn->dst_reg].off = aux->map_off; ++ if (map_value_has_spin_lock(map)) ++ regs[insn->dst_reg].id = ++env->id_gen; ++ } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { ++ regs[insn->dst_reg].type = CONST_PTR_TO_MAP; ++ } else { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } + +- regs[insn->dst_reg].type = CONST_PTR_TO_MAP; +- regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn); + return 0; + } + +@@ -1357,53 +6272,148 @@ static bool may_access_skb(enum bpf_prog + * Output: + * R0 - 8/16/32-bit skb data converted to cpu endianness + */ +-static int check_ld_abs(struct verifier_env *env, struct bpf_insn *insn) ++static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) + { +- struct reg_state *regs = env->cur_state.regs; ++ struct bpf_reg_state *regs = cur_regs(env); ++ static const int ctx_reg = BPF_REG_6; + u8 mode = BPF_MODE(insn->code); +- struct reg_state *reg; + int i, err; + + if (!may_access_skb(env->prog->type)) { +- verbose("BPF_LD_ABS|IND instructions not allowed for this program type\n"); ++ verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); ++ return -EINVAL; ++ } ++ ++ if (!env->ops->gen_ld_abs) { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } ++ ++ if (env->subprog_cnt > 1) { ++ /* when program has LD_ABS insn JITs and interpreter assume ++ * that r1 == ctx == skb which is not the case for callees ++ * that can have arbitrary arguments. It's problematic ++ * for main prog as well since JITs would need to analyze ++ * all functions in order to make proper register save/restore ++ * decisions in the main prog. Hence disallow LD_ABS with calls ++ */ ++ verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); + return -EINVAL; + } + + if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || + BPF_SIZE(insn->code) == BPF_DW || + (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { +- verbose("BPF_LD_ABS uses reserved fields\n"); ++ verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); + return -EINVAL; + } + + /* check whether implicit source operand (register R6) is readable */ +- err = check_reg_arg(regs, BPF_REG_6, SRC_OP); ++ err = check_reg_arg(env, ctx_reg, SRC_OP); + if (err) + return err; + +- if (regs[BPF_REG_6].type != PTR_TO_CTX) { +- verbose("at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); ++ /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as ++ * gen_ld_abs() may terminate the program at runtime, leading to ++ * reference leak. ++ */ ++ err = check_reference_leak(env); ++ if (err) { ++ verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); ++ return err; ++ } ++ ++ if (env->cur_state->active_spin_lock) { ++ verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); ++ return -EINVAL; ++ } ++ ++ if (regs[ctx_reg].type != PTR_TO_CTX) { ++ verbose(env, ++ "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); + return -EINVAL; + } + + if (mode == BPF_IND) { + /* check explicit source operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + } + ++ err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); ++ if (err < 0) ++ return err; ++ + /* reset caller saved regs to unreadable */ + for (i = 0; i < CALLER_SAVED_REGS; i++) { +- reg = regs + caller_saved[i]; +- reg->type = NOT_INIT; +- reg->imm = 0; ++ mark_reg_not_init(env, regs, caller_saved[i]); ++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); + } + + /* mark destination R0 register as readable, since it contains +- * the value fetched from the packet ++ * the value fetched from the packet. ++ * Already marked as written above. + */ +- regs[BPF_REG_0].type = UNKNOWN_VALUE; ++ mark_reg_unknown(env, regs, BPF_REG_0); ++ /* ld_abs load up to 32-bit skb data. */ ++ regs[BPF_REG_0].subreg_def = env->insn_idx + 1; ++ return 0; ++} ++ ++static int check_return_code(struct bpf_verifier_env *env) ++{ ++ struct tnum enforce_attach_type_range = tnum_unknown; ++ struct bpf_reg_state *reg; ++ struct tnum range = tnum_range(0, 1); ++ ++ switch (env->prog->type) { ++ case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: ++ if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || ++ env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG) ++ range = tnum_range(1, 1); ++ break; ++ case BPF_PROG_TYPE_CGROUP_SKB: ++ if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { ++ range = tnum_range(0, 3); ++ enforce_attach_type_range = tnum_range(2, 3); ++ } ++ break; ++ case BPF_PROG_TYPE_CGROUP_SOCK: ++ case BPF_PROG_TYPE_SOCK_OPS: ++ case BPF_PROG_TYPE_CGROUP_DEVICE: ++ case BPF_PROG_TYPE_CGROUP_SYSCTL: ++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: ++ break; ++ default: ++ return 0; ++ } ++ ++ reg = cur_regs(env) + BPF_REG_0; ++ if (reg->type != SCALAR_VALUE) { ++ verbose(env, "At program exit the register R0 is not a known value (%s)\n", ++ reg_type_str[reg->type]); ++ return -EINVAL; ++ } ++ ++ if (!tnum_in(range, reg->var_off)) { ++ char tn_buf[48]; ++ ++ verbose(env, "At program exit the register R0 "); ++ if (!tnum_is_unknown(reg->var_off)) { ++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); ++ verbose(env, "has value %s", tn_buf); ++ } else { ++ verbose(env, "has unknown scalar value"); ++ } ++ tnum_strn(tn_buf, sizeof(tn_buf), range); ++ verbose(env, " should have been in %s\n", tn_buf); ++ return -EINVAL; ++ } ++ ++ if (!tnum_is_unknown(enforce_attach_type_range) && ++ tnum_in(enforce_attach_type_range, reg->var_off)) ++ env->prog->enforce_expected_attach_type = 1; + return 0; + } + +@@ -1447,19 +6457,37 @@ enum { + BRANCH = 2, + }; + +-#define STATE_LIST_MARK ((struct verifier_state_list *) -1L) ++static u32 state_htab_size(struct bpf_verifier_env *env) ++{ ++ return env->prog->len; ++} ++ ++static struct bpf_verifier_state_list **explored_state( ++ struct bpf_verifier_env *env, ++ int idx) ++{ ++ struct bpf_verifier_state *cur = env->cur_state; ++ struct bpf_func_state *state = cur->frame[cur->curframe]; ++ ++ return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; ++} + +-static int *insn_stack; /* stack of insns to process */ +-static int cur_stack; /* current stack index */ +-static int *insn_state; ++static void init_explored_state(struct bpf_verifier_env *env, int idx) ++{ ++ env->insn_aux_data[idx].prune_point = true; ++} + + /* t, w, e - match pseudo-code above: + * t - index of current instruction + * w - next instruction + * e - edge + */ +-static int push_insn(int t, int w, int e, struct verifier_env *env) ++static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, ++ bool loop_ok) + { ++ int *insn_stack = env->cfg.insn_stack; ++ int *insn_state = env->cfg.insn_state; ++ + if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) + return 0; + +@@ -1467,30 +6495,35 @@ static int push_insn(int t, int w, int e + return 0; + + if (w < 0 || w >= env->prog->len) { +- verbose("jump out of range from insn %d to %d\n", t, w); ++ verbose_linfo(env, t, "%d: ", t); ++ verbose(env, "jump out of range from insn %d to %d\n", t, w); + return -EINVAL; + } + + if (e == BRANCH) + /* mark branch target for state pruning */ +- env->explored_states[w] = STATE_LIST_MARK; ++ init_explored_state(env, w); + + if (insn_state[w] == 0) { + /* tree-edge */ + insn_state[t] = DISCOVERED | e; + insn_state[w] = DISCOVERED; +- if (cur_stack >= env->prog->len) ++ if (env->cfg.cur_stack >= env->prog->len) + return -E2BIG; +- insn_stack[cur_stack++] = w; ++ insn_stack[env->cfg.cur_stack++] = w; + return 1; + } else if ((insn_state[w] & 0xF0) == DISCOVERED) { +- verbose("back-edge from insn %d to %d\n", t, w); ++ if (loop_ok && env->allow_ptr_leaks) ++ return 0; ++ verbose_linfo(env, t, "%d: ", t); ++ verbose_linfo(env, w, "%d: ", w); ++ verbose(env, "back-edge from insn %d to %d\n", t, w); + return -EINVAL; + } else if (insn_state[w] == EXPLORED) { + /* forward- or cross-edge */ + insn_state[t] = DISCOVERED | e; + } else { +- verbose("insn state internal bug\n"); ++ verbose(env, "insn state internal bug\n"); + return -EFAULT; + } + return 0; +@@ -1499,43 +6532,56 @@ static int push_insn(int t, int w, int e + /* non-recursive depth-first-search to detect loops in BPF program + * loop == back-edge in directed graph + */ +-static int check_cfg(struct verifier_env *env) ++static int check_cfg(struct bpf_verifier_env *env) + { + struct bpf_insn *insns = env->prog->insnsi; + int insn_cnt = env->prog->len; ++ int *insn_stack, *insn_state; + int ret = 0; + int i, t; + +- insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); ++ insn_state = env->cfg.insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); + if (!insn_state) + return -ENOMEM; + +- insn_stack = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); ++ insn_stack = env->cfg.insn_stack = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); + if (!insn_stack) { +- kfree(insn_state); ++ kvfree(insn_state); + return -ENOMEM; + } + + insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ + insn_stack[0] = 0; /* 0 is the first instruction */ +- cur_stack = 1; ++ env->cfg.cur_stack = 1; + + peek_stack: +- if (cur_stack == 0) ++ if (env->cfg.cur_stack == 0) + goto check_state; +- t = insn_stack[cur_stack - 1]; ++ t = insn_stack[env->cfg.cur_stack - 1]; + +- if (BPF_CLASS(insns[t].code) == BPF_JMP) { ++ if (BPF_CLASS(insns[t].code) == BPF_JMP || ++ BPF_CLASS(insns[t].code) == BPF_JMP32) { + u8 opcode = BPF_OP(insns[t].code); + + if (opcode == BPF_EXIT) { + goto mark_explored; + } else if (opcode == BPF_CALL) { +- ret = push_insn(t, t + 1, FALLTHROUGH, env); ++ ret = push_insn(t, t + 1, FALLTHROUGH, env, false); + if (ret == 1) + goto peek_stack; + else if (ret < 0) + goto err_free; ++ if (t + 1 < insn_cnt) ++ init_explored_state(env, t + 1); ++ if (insns[t].src_reg == BPF_PSEUDO_CALL) { ++ init_explored_state(env, t); ++ ret = push_insn(t, t + insns[t].imm + 1, BRANCH, ++ env, false); ++ if (ret == 1) ++ goto peek_stack; ++ else if (ret < 0) ++ goto err_free; ++ } + } else if (opcode == BPF_JA) { + if (BPF_SRC(insns[t].code) != BPF_K) { + ret = -EINVAL; +@@ -1543,25 +6589,31 @@ peek_stack: + } + /* unconditional jump with single edge */ + ret = push_insn(t, t + insns[t].off + 1, +- FALLTHROUGH, env); ++ FALLTHROUGH, env, true); + if (ret == 1) + goto peek_stack; + else if (ret < 0) + goto err_free; ++ /* unconditional jmp is not a good pruning point, ++ * but it's marked, since backtracking needs ++ * to record jmp history in is_state_visited(). ++ */ ++ init_explored_state(env, t + insns[t].off + 1); + /* tell verifier to check for equivalent states + * after every call and jump + */ + if (t + 1 < insn_cnt) +- env->explored_states[t + 1] = STATE_LIST_MARK; ++ init_explored_state(env, t + 1); + } else { + /* conditional jump with two edges */ +- ret = push_insn(t, t + 1, FALLTHROUGH, env); ++ init_explored_state(env, t); ++ ret = push_insn(t, t + 1, FALLTHROUGH, env, true); + if (ret == 1) + goto peek_stack; + else if (ret < 0) + goto err_free; + +- ret = push_insn(t, t + insns[t].off + 1, BRANCH, env); ++ ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true); + if (ret == 1) + goto peek_stack; + else if (ret < 0) +@@ -1571,7 +6623,7 @@ peek_stack: + /* all other non-branch instructions with single + * fall-through edge + */ +- ret = push_insn(t, t + 1, FALLTHROUGH, env); ++ ret = push_insn(t, t + 1, FALLTHROUGH, env, false); + if (ret == 1) + goto peek_stack; + else if (ret < 0) +@@ -1580,8 +6632,8 @@ peek_stack: + + mark_explored: + insn_state[t] = EXPLORED; +- if (cur_stack-- <= 0) { +- verbose("pop stack internal bug\n"); ++ if (env->cfg.cur_stack-- <= 0) { ++ verbose(env, "pop stack internal bug\n"); + ret = -EFAULT; + goto err_free; + } +@@ -1590,7 +6642,7 @@ mark_explored: + check_state: + for (i = 0; i < insn_cnt; i++) { + if (insn_state[i] != EXPLORED) { +- verbose("unreachable insn %d\n", i); ++ verbose(env, "unreachable insn %d\n", i); + ret = -EINVAL; + goto err_free; + } +@@ -1598,11 +6650,616 @@ check_state: + ret = 0; /* cfg looks good */ + + err_free: +- kfree(insn_state); +- kfree(insn_stack); ++ kvfree(insn_state); ++ kvfree(insn_stack); ++ env->cfg.insn_state = env->cfg.insn_stack = NULL; + return ret; + } + ++/* The minimum supported BTF func info size */ ++#define MIN_BPF_FUNCINFO_SIZE 8 ++#define MAX_FUNCINFO_REC_SIZE 252 ++ ++static int check_btf_func(struct bpf_verifier_env *env, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ u32 i, nfuncs, urec_size, min_size; ++ u32 krec_size = sizeof(struct bpf_func_info); ++ struct bpf_func_info *krecord; ++ const struct btf_type *type; ++ struct bpf_prog *prog; ++ const struct btf *btf; ++ void __user *urecord; ++ u32 prev_offset = 0; ++ int ret = 0; ++ ++ nfuncs = attr->func_info_cnt; ++ if (!nfuncs) ++ return 0; ++ ++ if (nfuncs != env->subprog_cnt) { ++ verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); ++ return -EINVAL; ++ } ++ ++ urec_size = attr->func_info_rec_size; ++ if (urec_size < MIN_BPF_FUNCINFO_SIZE || ++ urec_size > MAX_FUNCINFO_REC_SIZE || ++ urec_size % sizeof(u32)) { ++ verbose(env, "invalid func info rec size %u\n", urec_size); ++ return -EINVAL; ++ } ++ ++ prog = env->prog; ++ btf = prog->aux->btf; ++ ++ urecord = u64_to_user_ptr(attr->func_info); ++ min_size = min_t(u32, krec_size, urec_size); ++ ++ krecord = kcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); ++ if (!krecord) ++ return -ENOMEM; ++ ++ for (i = 0; i < nfuncs; i++) { ++ ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); ++ if (ret) { ++ if (ret == -E2BIG) { ++ verbose(env, "nonzero tailing record in func info"); ++ /* set the size kernel expects so loader can zero ++ * out the rest of the record. ++ */ ++ if (put_user(min_size, &uattr->func_info_rec_size)) ++ ret = -EFAULT; ++ } ++ goto err_free; ++ } ++ ++ if (copy_from_user(&krecord[i], urecord, min_size)) { ++ ret = -EFAULT; ++ goto err_free; ++ } ++ ++ /* check insn_off */ ++ if (i == 0) { ++ if (krecord[i].insn_off) { ++ verbose(env, ++ "nonzero insn_off %u for the first func info record", ++ krecord[i].insn_off); ++ ret = -EINVAL; ++ goto err_free; ++ } ++ } else if (krecord[i].insn_off <= prev_offset) { ++ verbose(env, ++ "same or smaller insn offset (%u) than previous func info record (%u)", ++ krecord[i].insn_off, prev_offset); ++ ret = -EINVAL; ++ goto err_free; ++ } ++ ++ if (env->subprog_info[i].start != krecord[i].insn_off) { ++ verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); ++ ret = -EINVAL; ++ goto err_free; ++ } ++ ++ /* check type_id */ ++ type = btf_type_by_id(btf, krecord[i].type_id); ++ if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) { ++ verbose(env, "invalid type id %d in func info", ++ krecord[i].type_id); ++ ret = -EINVAL; ++ goto err_free; ++ } ++ ++ prev_offset = krecord[i].insn_off; ++ urecord += urec_size; ++ } ++ ++ prog->aux->func_info = krecord; ++ prog->aux->func_info_cnt = nfuncs; ++ return 0; ++ ++err_free: ++ kvfree(krecord); ++ return ret; ++} ++ ++static void adjust_btf_func(struct bpf_verifier_env *env) ++{ ++ int i; ++ ++ if (!env->prog->aux->func_info) ++ return; ++ ++ for (i = 0; i < env->subprog_cnt; i++) ++ env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; ++} ++ ++#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ ++ sizeof(((struct bpf_line_info *)(0))->line_col)) ++#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE ++ ++static int check_btf_line(struct bpf_verifier_env *env, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; ++ struct bpf_subprog_info *sub; ++ struct bpf_line_info *linfo; ++ struct bpf_prog *prog; ++ const struct btf *btf; ++ void __user *ulinfo; ++ int err; ++ ++ nr_linfo = attr->line_info_cnt; ++ if (!nr_linfo) ++ return 0; ++ ++ rec_size = attr->line_info_rec_size; ++ if (rec_size < MIN_BPF_LINEINFO_SIZE || ++ rec_size > MAX_LINEINFO_REC_SIZE || ++ rec_size & (sizeof(u32) - 1)) ++ return -EINVAL; ++ ++ /* Need to zero it in case the userspace may ++ * pass in a smaller bpf_line_info object. ++ */ ++ linfo = kcalloc(nr_linfo, sizeof(struct bpf_line_info), ++ GFP_KERNEL | __GFP_NOWARN); ++ if (!linfo) ++ return -ENOMEM; ++ ++ prog = env->prog; ++ btf = prog->aux->btf; ++ ++ s = 0; ++ sub = env->subprog_info; ++ ulinfo = u64_to_user_ptr(attr->line_info); ++ expected_size = sizeof(struct bpf_line_info); ++ ncopy = min_t(u32, expected_size, rec_size); ++ for (i = 0; i < nr_linfo; i++) { ++ err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); ++ if (err) { ++ if (err == -E2BIG) { ++ verbose(env, "nonzero tailing record in line_info"); ++ if (put_user(expected_size, ++ &uattr->line_info_rec_size)) ++ err = -EFAULT; ++ } ++ goto err_free; ++ } ++ ++ if (copy_from_user(&linfo[i], ulinfo, ncopy)) { ++ err = -EFAULT; ++ goto err_free; ++ } ++ ++ /* ++ * Check insn_off to ensure ++ * 1) strictly increasing AND ++ * 2) bounded by prog->len ++ * ++ * The linfo[0].insn_off == 0 check logically falls into ++ * the later "missing bpf_line_info for func..." case ++ * because the first linfo[0].insn_off must be the ++ * first sub also and the first sub must have ++ * subprog_info[0].start == 0. ++ */ ++ if ((i && linfo[i].insn_off <= prev_offset) || ++ linfo[i].insn_off >= prog->len) { ++ verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", ++ i, linfo[i].insn_off, prev_offset, ++ prog->len); ++ err = -EINVAL; ++ goto err_free; ++ } ++ ++ if (!prog->insnsi[linfo[i].insn_off].code) { ++ verbose(env, ++ "Invalid insn code at line_info[%u].insn_off\n", ++ i); ++ err = -EINVAL; ++ goto err_free; ++ } ++ ++ if (!btf_name_by_offset(btf, linfo[i].line_off) || ++ !btf_name_by_offset(btf, linfo[i].file_name_off)) { ++ verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); ++ err = -EINVAL; ++ goto err_free; ++ } ++ ++ if (s != env->subprog_cnt) { ++ if (linfo[i].insn_off == sub[s].start) { ++ sub[s].linfo_idx = i; ++ s++; ++ } else if (sub[s].start < linfo[i].insn_off) { ++ verbose(env, "missing bpf_line_info for func#%u\n", s); ++ err = -EINVAL; ++ goto err_free; ++ } ++ } ++ ++ prev_offset = linfo[i].insn_off; ++ ulinfo += rec_size; ++ } ++ ++ if (s != env->subprog_cnt) { ++ verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", ++ env->subprog_cnt - s, s); ++ err = -EINVAL; ++ goto err_free; ++ } ++ ++ prog->aux->linfo = linfo; ++ prog->aux->nr_linfo = nr_linfo; ++ ++ return 0; ++ ++err_free: ++ kvfree(linfo); ++ return err; ++} ++ ++static int check_btf_info(struct bpf_verifier_env *env, ++ const union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ struct btf *btf; ++ int err; ++ ++ if (!attr->func_info_cnt && !attr->line_info_cnt) ++ return 0; ++ ++ btf = btf_get_by_fd(attr->prog_btf_fd); ++ if (IS_ERR(btf)) ++ return PTR_ERR(btf); ++ env->prog->aux->btf = btf; ++ ++ err = check_btf_func(env, attr, uattr); ++ if (err) ++ return err; ++ ++ err = check_btf_line(env, attr, uattr); ++ if (err) ++ return err; ++ ++ return 0; ++} ++ ++/* check %cur's range satisfies %old's */ ++static bool range_within(struct bpf_reg_state *old, ++ struct bpf_reg_state *cur) ++{ ++ return old->umin_value <= cur->umin_value && ++ old->umax_value >= cur->umax_value && ++ old->smin_value <= cur->smin_value && ++ old->smax_value >= cur->smax_value; ++} ++ ++/* Maximum number of register states that can exist at once */ ++#define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) ++struct idpair { ++ u32 old; ++ u32 cur; ++}; ++ ++/* If in the old state two registers had the same id, then they need to have ++ * the same id in the new state as well. But that id could be different from ++ * the old state, so we need to track the mapping from old to new ids. ++ * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent ++ * regs with old id 5 must also have new id 9 for the new state to be safe. But ++ * regs with a different old id could still have new id 9, we don't care about ++ * that. ++ * So we look through our idmap to see if this old id has been seen before. If ++ * so, we require the new id to match; otherwise, we add the id pair to the map. ++ */ ++static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) ++{ ++ unsigned int i; ++ ++ for (i = 0; i < ID_MAP_SIZE; i++) { ++ if (!idmap[i].old) { ++ /* Reached an empty slot; haven't seen this id before */ ++ idmap[i].old = old_id; ++ idmap[i].cur = cur_id; ++ return true; ++ } ++ if (idmap[i].old == old_id) ++ return idmap[i].cur == cur_id; ++ } ++ /* We ran out of idmap slots, which should be impossible */ ++ WARN_ON_ONCE(1); ++ return false; ++} ++ ++static void clean_func_state(struct bpf_verifier_env *env, ++ struct bpf_func_state *st) ++{ ++ enum bpf_reg_liveness live; ++ int i, j; ++ ++ for (i = 0; i < BPF_REG_FP; i++) { ++ live = st->regs[i].live; ++ /* liveness must not touch this register anymore */ ++ st->regs[i].live |= REG_LIVE_DONE; ++ if (!(live & REG_LIVE_READ)) ++ /* since the register is unused, clear its state ++ * to make further comparison simpler ++ */ ++ __mark_reg_not_init(env, &st->regs[i]); ++ } ++ ++ for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { ++ live = st->stack[i].spilled_ptr.live; ++ /* liveness must not touch this stack slot anymore */ ++ st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; ++ if (!(live & REG_LIVE_READ)) { ++ __mark_reg_not_init(env, &st->stack[i].spilled_ptr); ++ for (j = 0; j < BPF_REG_SIZE; j++) ++ st->stack[i].slot_type[j] = STACK_INVALID; ++ } ++ } ++} ++ ++static void clean_verifier_state(struct bpf_verifier_env *env, ++ struct bpf_verifier_state *st) ++{ ++ int i; ++ ++ if (st->frame[0]->regs[0].live & REG_LIVE_DONE) ++ /* all regs in this state in all frames were already marked */ ++ return; ++ ++ for (i = 0; i <= st->curframe; i++) ++ clean_func_state(env, st->frame[i]); ++} ++ ++/* the parentage chains form a tree. ++ * the verifier states are added to state lists at given insn and ++ * pushed into state stack for future exploration. ++ * when the verifier reaches bpf_exit insn some of the verifer states ++ * stored in the state lists have their final liveness state already, ++ * but a lot of states will get revised from liveness point of view when ++ * the verifier explores other branches. ++ * Example: ++ * 1: r0 = 1 ++ * 2: if r1 == 100 goto pc+1 ++ * 3: r0 = 2 ++ * 4: exit ++ * when the verifier reaches exit insn the register r0 in the state list of ++ * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch ++ * of insn 2 and goes exploring further. At the insn 4 it will walk the ++ * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. ++ * ++ * Since the verifier pushes the branch states as it sees them while exploring ++ * the program the condition of walking the branch instruction for the second ++ * time means that all states below this branch were already explored and ++ * their final liveness markes are already propagated. ++ * Hence when the verifier completes the search of state list in is_state_visited() ++ * we can call this clean_live_states() function to mark all liveness states ++ * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' ++ * will not be used. ++ * This function also clears the registers and stack for states that !READ ++ * to simplify state merging. ++ * ++ * Important note here that walking the same branch instruction in the callee ++ * doesn't meant that the states are DONE. The verifier has to compare ++ * the callsites ++ */ ++static void clean_live_states(struct bpf_verifier_env *env, int insn, ++ struct bpf_verifier_state *cur) ++{ ++ struct bpf_verifier_state_list *sl; ++ int i; ++ ++ sl = *explored_state(env, insn); ++ while (sl) { ++ if (sl->state.branches) ++ goto next; ++ if (sl->state.insn_idx != insn || ++ sl->state.curframe != cur->curframe) ++ goto next; ++ for (i = 0; i <= cur->curframe; i++) ++ if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) ++ goto next; ++ clean_verifier_state(env, &sl->state); ++next: ++ sl = sl->next; ++ } ++} ++ ++/* Returns true if (rold safe implies rcur safe) */ ++static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, ++ struct idpair *idmap) ++{ ++ bool equal; ++ ++ if (!(rold->live & REG_LIVE_READ)) ++ /* explored state didn't use this */ ++ return true; ++ ++ equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; ++ ++ if (rold->type == PTR_TO_STACK) ++ /* two stack pointers are equal only if they're pointing to ++ * the same stack frame, since fp-8 in foo != fp-8 in bar ++ */ ++ return equal && rold->frameno == rcur->frameno; ++ ++ if (equal) ++ return true; ++ ++ if (rold->type == NOT_INIT) ++ /* explored state can't have used this */ ++ return true; ++ if (rcur->type == NOT_INIT) ++ return false; ++ switch (rold->type) { ++ case SCALAR_VALUE: ++ if (rcur->type == SCALAR_VALUE) { ++ if (!rold->precise && !rcur->precise) ++ return true; ++ /* new val must satisfy old val knowledge */ ++ return range_within(rold, rcur) && ++ tnum_in(rold->var_off, rcur->var_off); ++ } else { ++ /* We're trying to use a pointer in place of a scalar. ++ * Even if the scalar was unbounded, this could lead to ++ * pointer leaks because scalars are allowed to leak ++ * while pointers are not. We could make this safe in ++ * special cases if root is calling us, but it's ++ * probably not worth the hassle. ++ */ ++ return false; ++ } ++ case PTR_TO_MAP_VALUE: ++ /* If the new min/max/var_off satisfy the old ones and ++ * everything else matches, we are OK. ++ * 'id' is not compared, since it's only used for maps with ++ * bpf_spin_lock inside map element and in such cases if ++ * the rest of the prog is valid for one map element then ++ * it's valid for all map elements regardless of the key ++ * used in bpf_map_lookup() ++ */ ++ return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && ++ range_within(rold, rcur) && ++ tnum_in(rold->var_off, rcur->var_off); ++ case PTR_TO_MAP_VALUE_OR_NULL: ++ /* a PTR_TO_MAP_VALUE could be safe to use as a ++ * PTR_TO_MAP_VALUE_OR_NULL into the same map. ++ * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- ++ * checked, doing so could have affected others with the same ++ * id, and we can't check for that because we lost the id when ++ * we converted to a PTR_TO_MAP_VALUE. ++ */ ++ if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) ++ return false; ++ if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) ++ return false; ++ /* Check our ids match any regs they're supposed to */ ++ return check_ids(rold->id, rcur->id, idmap); ++ case PTR_TO_PACKET_META: ++ case PTR_TO_PACKET: ++ if (rcur->type != rold->type) ++ return false; ++ /* We must have at least as much range as the old ptr ++ * did, so that any accesses which were safe before are ++ * still safe. This is true even if old range < old off, ++ * since someone could have accessed through (ptr - k), or ++ * even done ptr -= k in a register, to get a safe access. ++ */ ++ if (rold->range > rcur->range) ++ return false; ++ /* If the offsets don't match, we can't trust our alignment; ++ * nor can we be sure that we won't fall out of range. ++ */ ++ if (rold->off != rcur->off) ++ return false; ++ /* id relations must be preserved */ ++ if (rold->id && !check_ids(rold->id, rcur->id, idmap)) ++ return false; ++ /* new val must satisfy old val knowledge */ ++ return range_within(rold, rcur) && ++ tnum_in(rold->var_off, rcur->var_off); ++ case PTR_TO_CTX: ++ case CONST_PTR_TO_MAP: ++ case PTR_TO_PACKET_END: ++ case PTR_TO_FLOW_KEYS: ++ case PTR_TO_SOCKET: ++ case PTR_TO_SOCKET_OR_NULL: ++ case PTR_TO_SOCK_COMMON: ++ case PTR_TO_SOCK_COMMON_OR_NULL: ++ case PTR_TO_TCP_SOCK: ++ case PTR_TO_TCP_SOCK_OR_NULL: ++ case PTR_TO_XDP_SOCK: ++ /* Only valid matches are exact, which memcmp() above ++ * would have accepted ++ */ ++ default: ++ /* Don't know what's going on, just say it's not safe */ ++ return false; ++ } ++ ++ /* Shouldn't get here; if we do, say it's not safe */ ++ WARN_ON_ONCE(1); ++ return false; ++} ++ ++static bool stacksafe(struct bpf_func_state *old, ++ struct bpf_func_state *cur, ++ struct idpair *idmap) ++{ ++ int i, spi; ++ ++ /* walk slots of the explored stack and ignore any additional ++ * slots in the current stack, since explored(safe) state ++ * didn't use them ++ */ ++ for (i = 0; i < old->allocated_stack; i++) { ++ spi = i / BPF_REG_SIZE; ++ ++ if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { ++ i += BPF_REG_SIZE - 1; ++ /* explored state didn't use this */ ++ continue; ++ } ++ ++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) ++ continue; ++ ++ /* explored stack has more populated slots than current stack ++ * and these slots were used ++ */ ++ if (i >= cur->allocated_stack) ++ return false; ++ ++ /* if old state was safe with misc data in the stack ++ * it will be safe with zero-initialized stack. ++ * The opposite is not true ++ */ ++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && ++ cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) ++ continue; ++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != ++ cur->stack[spi].slot_type[i % BPF_REG_SIZE]) ++ /* Ex: old explored (safe) state has STACK_SPILL in ++ * this stack slot, but current has has STACK_MISC -> ++ * this verifier states are not equivalent, ++ * return false to continue verification of this path ++ */ ++ return false; ++ if (i % BPF_REG_SIZE) ++ continue; ++ if (old->stack[spi].slot_type[0] != STACK_SPILL) ++ continue; ++ if (!regsafe(&old->stack[spi].spilled_ptr, ++ &cur->stack[spi].spilled_ptr, ++ idmap)) ++ /* when explored and current stack slot are both storing ++ * spilled registers, check that stored pointers types ++ * are the same as well. ++ * Ex: explored safe path could have stored ++ * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} ++ * but current path has stored: ++ * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} ++ * such verifier states are not equivalent. ++ * return false to continue verification of this path ++ */ ++ return false; ++ } ++ return true; ++} ++ ++static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) ++{ ++ if (old->acquired_refs != cur->acquired_refs) ++ return false; ++ return !memcmp(old->refs, cur->refs, ++ sizeof(*old->refs) * old->acquired_refs); ++} ++ + /* compare two verifier states + * + * all states stored in state_list are known to be valid, since +@@ -1629,165 +7286,562 @@ err_free: + * whereas register type in current state is meaningful, it means that + * the current state will reach 'bpf_exit' instruction safely + */ +-static bool states_equal(struct verifier_state *old, struct verifier_state *cur) ++static bool func_states_equal(struct bpf_func_state *old, ++ struct bpf_func_state *cur) + { ++ struct idpair *idmap; ++ bool ret = false; + int i; + ++ idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); ++ /* If we failed to allocate the idmap, just say it's not safe */ ++ if (!idmap) ++ return false; ++ + for (i = 0; i < MAX_BPF_REG; i++) { +- if (memcmp(&old->regs[i], &cur->regs[i], +- sizeof(old->regs[0])) != 0) { +- if (old->regs[i].type == NOT_INIT || +- (old->regs[i].type == UNKNOWN_VALUE && +- cur->regs[i].type != NOT_INIT)) +- continue; ++ if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) ++ goto out_free; ++ } ++ ++ if (!stacksafe(old, cur, idmap)) ++ goto out_free; ++ ++ if (!refsafe(old, cur)) ++ goto out_free; ++ ret = true; ++out_free: ++ kfree(idmap); ++ return ret; ++} ++ ++static bool states_equal(struct bpf_verifier_env *env, ++ struct bpf_verifier_state *old, ++ struct bpf_verifier_state *cur) ++{ ++ int i; ++ ++ if (old->curframe != cur->curframe) ++ return false; ++ ++ /* Verification state from speculative execution simulation ++ * must never prune a non-speculative execution one. ++ */ ++ if (old->speculative && !cur->speculative) ++ return false; ++ ++ if (old->active_spin_lock != cur->active_spin_lock) ++ return false; ++ ++ /* for states to be equal callsites have to be the same ++ * and all frame states need to be equivalent ++ */ ++ for (i = 0; i <= old->curframe; i++) { ++ if (old->frame[i]->callsite != cur->frame[i]->callsite) ++ return false; ++ if (!func_states_equal(old->frame[i], cur->frame[i])) + return false; ++ } ++ return true; ++} ++ ++/* Return 0 if no propagation happened. Return negative error code if error ++ * happened. Otherwise, return the propagated bit. ++ */ ++static int propagate_liveness_reg(struct bpf_verifier_env *env, ++ struct bpf_reg_state *reg, ++ struct bpf_reg_state *parent_reg) ++{ ++ u8 parent_flag = parent_reg->live & REG_LIVE_READ; ++ u8 flag = reg->live & REG_LIVE_READ; ++ int err; ++ ++ /* When comes here, read flags of PARENT_REG or REG could be any of ++ * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need ++ * of propagation if PARENT_REG has strongest REG_LIVE_READ64. ++ */ ++ if (parent_flag == REG_LIVE_READ64 || ++ /* Or if there is no read flag from REG. */ ++ !flag || ++ /* Or if the read flag from REG is the same as PARENT_REG. */ ++ parent_flag == flag) ++ return 0; ++ ++ err = mark_reg_read(env, reg, parent_reg, flag); ++ if (err) ++ return err; ++ ++ return flag; ++} ++ ++/* A write screens off any subsequent reads; but write marks come from the ++ * straight-line code between a state and its parent. When we arrive at an ++ * equivalent state (jump target or such) we didn't arrive by the straight-line ++ * code, so read marks in the state must propagate to the parent regardless ++ * of the state's write marks. That's what 'parent == state->parent' comparison ++ * in mark_reg_read() is for. ++ */ ++static int propagate_liveness(struct bpf_verifier_env *env, ++ const struct bpf_verifier_state *vstate, ++ struct bpf_verifier_state *vparent) ++{ ++ struct bpf_reg_state *state_reg, *parent_reg; ++ struct bpf_func_state *state, *parent; ++ int i, frame, err = 0; ++ ++ if (vparent->curframe != vstate->curframe) { ++ WARN(1, "propagate_live: parent frame %d current frame %d\n", ++ vparent->curframe, vstate->curframe); ++ return -EFAULT; ++ } ++ /* Propagate read liveness of registers... */ ++ BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); ++ for (frame = 0; frame <= vstate->curframe; frame++) { ++ parent = vparent->frame[frame]; ++ state = vstate->frame[frame]; ++ parent_reg = parent->regs; ++ state_reg = state->regs; ++ /* We don't need to worry about FP liveness, it's read-only */ ++ for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { ++ err = propagate_liveness_reg(env, &state_reg[i], ++ &parent_reg[i]); ++ if (err < 0) ++ return err; ++ if (err == REG_LIVE_READ64) ++ mark_insn_zext(env, &parent_reg[i]); ++ } ++ ++ /* Propagate stack slots. */ ++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && ++ i < parent->allocated_stack / BPF_REG_SIZE; i++) { ++ parent_reg = &parent->stack[i].spilled_ptr; ++ state_reg = &state->stack[i].spilled_ptr; ++ err = propagate_liveness_reg(env, state_reg, ++ parent_reg); ++ if (err < 0) ++ return err; + } + } ++ return 0; ++} + +- for (i = 0; i < MAX_BPF_STACK; i++) { +- if (old->stack_slot_type[i] == STACK_INVALID) ++/* find precise scalars in the previous equivalent state and ++ * propagate them into the current state ++ */ ++static int propagate_precision(struct bpf_verifier_env *env, ++ const struct bpf_verifier_state *old) ++{ ++ struct bpf_reg_state *state_reg; ++ struct bpf_func_state *state; ++ int i, err = 0; ++ ++ state = old->frame[old->curframe]; ++ state_reg = state->regs; ++ for (i = 0; i < BPF_REG_FP; i++, state_reg++) { ++ if (state_reg->type != SCALAR_VALUE || ++ !state_reg->precise) + continue; +- if (old->stack_slot_type[i] != cur->stack_slot_type[i]) +- /* Ex: old explored (safe) state has STACK_SPILL in +- * this stack slot, but current has has STACK_MISC -> +- * this verifier states are not equivalent, +- * return false to continue verification of this path +- */ +- return false; +- if (i % BPF_REG_SIZE) ++ if (env->log.level & BPF_LOG_LEVEL2) ++ verbose(env, "propagating r%d\n", i); ++ err = mark_chain_precision(env, i); ++ if (err < 0) ++ return err; ++ } ++ ++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { ++ if (state->stack[i].slot_type[0] != STACK_SPILL) + continue; +- if (memcmp(&old->spilled_regs[i / BPF_REG_SIZE], +- &cur->spilled_regs[i / BPF_REG_SIZE], +- sizeof(old->spilled_regs[0]))) +- /* when explored and current stack slot types are +- * the same, check that stored pointers types +- * are the same as well. +- * Ex: explored safe path could have stored +- * (struct reg_state) {.type = PTR_TO_STACK, .imm = -8} +- * but current path has stored: +- * (struct reg_state) {.type = PTR_TO_STACK, .imm = -16} +- * such verifier states are not equivalent. +- * return false to continue verification of this path +- */ +- return false; +- else ++ state_reg = &state->stack[i].spilled_ptr; ++ if (state_reg->type != SCALAR_VALUE || ++ !state_reg->precise) + continue; ++ if (env->log.level & BPF_LOG_LEVEL2) ++ verbose(env, "propagating fp%d\n", ++ (-i - 1) * BPF_REG_SIZE); ++ err = mark_chain_precision_stack(env, i); ++ if (err < 0) ++ return err; + } ++ return 0; ++} ++ ++static bool states_maybe_looping(struct bpf_verifier_state *old, ++ struct bpf_verifier_state *cur) ++{ ++ struct bpf_func_state *fold, *fcur; ++ int i, fr = cur->curframe; ++ ++ if (old->curframe != fr) ++ return false; ++ ++ fold = old->frame[fr]; ++ fcur = cur->frame[fr]; ++ for (i = 0; i < MAX_BPF_REG; i++) ++ if (memcmp(&fold->regs[i], &fcur->regs[i], ++ offsetof(struct bpf_reg_state, parent))) ++ return false; + return true; + } + +-static int is_state_visited(struct verifier_env *env, int insn_idx) ++ ++static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) + { +- struct verifier_state_list *new_sl; +- struct verifier_state_list *sl; ++ struct bpf_verifier_state_list *new_sl; ++ struct bpf_verifier_state_list *sl, **pprev; ++ struct bpf_verifier_state *cur = env->cur_state, *new; ++ int i, j, err, states_cnt = 0; ++ bool add_new_state = env->test_state_freq ? true : false; + +- sl = env->explored_states[insn_idx]; +- if (!sl) ++ cur->last_insn_idx = env->prev_insn_idx; ++ if (!env->insn_aux_data[insn_idx].prune_point) + /* this 'insn_idx' instruction wasn't marked, so we will not + * be doing state search here + */ + return 0; + +- while (sl != STATE_LIST_MARK) { +- if (states_equal(&sl->state, &env->cur_state)) ++ /* bpf progs typically have pruning point every 4 instructions ++ * http://vger.kernel.org/bpfconf2019.html#session-1 ++ * Do not add new state for future pruning if the verifier hasn't seen ++ * at least 2 jumps and at least 8 instructions. ++ * This heuristics helps decrease 'total_states' and 'peak_states' metric. ++ * In tests that amounts to up to 50% reduction into total verifier ++ * memory consumption and 20% verifier time speedup. ++ */ ++ if (env->jmps_processed - env->prev_jmps_processed >= 2 && ++ env->insn_processed - env->prev_insn_processed >= 8) ++ add_new_state = true; ++ ++ pprev = explored_state(env, insn_idx); ++ sl = *pprev; ++ ++ clean_live_states(env, insn_idx, cur); ++ ++ while (sl) { ++ states_cnt++; ++ if (sl->state.insn_idx != insn_idx) ++ goto next; ++ if (sl->state.branches) { ++ if (states_maybe_looping(&sl->state, cur) && ++ states_equal(env, &sl->state, cur)) { ++ verbose_linfo(env, insn_idx, "; "); ++ verbose(env, "infinite loop detected at insn %d\n", insn_idx); ++ return -EINVAL; ++ } ++ /* if the verifier is processing a loop, avoid adding new state ++ * too often, since different loop iterations have distinct ++ * states and may not help future pruning. ++ * This threshold shouldn't be too low to make sure that ++ * a loop with large bound will be rejected quickly. ++ * The most abusive loop will be: ++ * r1 += 1 ++ * if r1 < 1000000 goto pc-2 ++ * 1M insn_procssed limit / 100 == 10k peak states. ++ * This threshold shouldn't be too high either, since states ++ * at the end of the loop are likely to be useful in pruning. ++ */ ++ if (env->jmps_processed - env->prev_jmps_processed < 20 && ++ env->insn_processed - env->prev_insn_processed < 100) ++ add_new_state = false; ++ goto miss; ++ } ++ if (states_equal(env, &sl->state, cur)) { ++ sl->hit_cnt++; + /* reached equivalent register/stack state, +- * prune the search ++ * prune the search. ++ * Registers read by the continuation are read by us. ++ * If we have any write marks in env->cur_state, they ++ * will prevent corresponding reads in the continuation ++ * from reaching our parent (an explored_state). Our ++ * own state will get the read marks recorded, but ++ * they'll be immediately forgotten as we're pruning ++ * this state and will pop a new one. + */ +- return 1; +- sl = sl->next; +- } ++ err = propagate_liveness(env, &sl->state, cur); + +- /* there were no equivalent states, remember current one. +- * technically the current state is not proven to be safe yet, +- * but it will either reach bpf_exit (which means it's safe) or +- * it will be rejected. Since there are no loops, we won't be +- * seeing this 'insn_idx' instruction again on the way to bpf_exit ++ /* if previous state reached the exit with precision and ++ * current state is equivalent to it (except precsion marks) ++ * the precision needs to be propagated back in ++ * the current state. ++ */ ++ err = err ? : push_jmp_history(env, cur); ++ err = err ? : propagate_precision(env, &sl->state); ++ if (err) ++ return err; ++ return 1; ++ } ++miss: ++ /* when new state is not going to be added do not increase miss count. ++ * Otherwise several loop iterations will remove the state ++ * recorded earlier. The goal of these heuristics is to have ++ * states from some iterations of the loop (some in the beginning ++ * and some at the end) to help pruning. ++ */ ++ if (add_new_state) ++ sl->miss_cnt++; ++ /* heuristic to determine whether this state is beneficial ++ * to keep checking from state equivalence point of view. ++ * Higher numbers increase max_states_per_insn and verification time, ++ * but do not meaningfully decrease insn_processed. ++ */ ++ if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { ++ /* the state is unlikely to be useful. Remove it to ++ * speed up verification ++ */ ++ *pprev = sl->next; ++ if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { ++ u32 br = sl->state.branches; ++ ++ WARN_ONCE(br, ++ "BUG live_done but branches_to_explore %d\n", ++ br); ++ free_verifier_state(&sl->state, false); ++ kfree(sl); ++ env->peak_states--; ++ } else { ++ /* cannot free this state, since parentage chain may ++ * walk it later. Add it for free_list instead to ++ * be freed at the end of verification ++ */ ++ sl->next = env->free_list; ++ env->free_list = sl; ++ } ++ sl = *pprev; ++ continue; ++ } ++next: ++ pprev = &sl->next; ++ sl = *pprev; ++ } ++ ++ if (env->max_states_per_insn < states_cnt) ++ env->max_states_per_insn = states_cnt; ++ ++ if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) ++ return push_jmp_history(env, cur); ++ ++ if (!add_new_state) ++ return push_jmp_history(env, cur); ++ ++ /* There were no equivalent states, remember the current one. ++ * Technically the current state is not proven to be safe yet, ++ * but it will either reach outer most bpf_exit (which means it's safe) ++ * or it will be rejected. When there are no loops the verifier won't be ++ * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) ++ * again on the way to bpf_exit. ++ * When looping the sl->state.branches will be > 0 and this state ++ * will not be considered for equivalence until branches == 0. + */ +- new_sl = kmalloc(sizeof(struct verifier_state_list), GFP_USER); ++ new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); + if (!new_sl) + return -ENOMEM; ++ env->total_states++; ++ env->peak_states++; ++ env->prev_jmps_processed = env->jmps_processed; ++ env->prev_insn_processed = env->insn_processed; + + /* add new state to the head of linked list */ +- memcpy(&new_sl->state, &env->cur_state, sizeof(env->cur_state)); +- new_sl->next = env->explored_states[insn_idx]; +- env->explored_states[insn_idx] = new_sl; ++ new = &new_sl->state; ++ err = copy_verifier_state(new, cur); ++ if (err) { ++ free_verifier_state(new, false); ++ kfree(new_sl); ++ return err; ++ } ++ new->insn_idx = insn_idx; ++ WARN_ONCE(new->branches != 1, ++ "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); ++ ++ cur->parent = new; ++ cur->first_insn_idx = insn_idx; ++ clear_jmp_history(cur); ++ new_sl->next = *explored_state(env, insn_idx); ++ *explored_state(env, insn_idx) = new_sl; ++ /* connect new state to parentage chain. Current frame needs all ++ * registers connected. Only r6 - r9 of the callers are alive (pushed ++ * to the stack implicitly by JITs) so in callers' frames connect just ++ * r6 - r9 as an optimization. Callers will have r1 - r5 connected to ++ * the state of the call instruction (with WRITTEN set), and r0 comes ++ * from callee with its full parentage chain, anyway. ++ */ ++ /* clear write marks in current state: the writes we did are not writes ++ * our child did, so they don't screen off its reads from us. ++ * (There are no read marks in current state, because reads always mark ++ * their parent and current state never has children yet. Only ++ * explored_states can get read marks.) ++ */ ++ for (j = 0; j <= cur->curframe; j++) { ++ for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) ++ cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; ++ for (i = 0; i < BPF_REG_FP; i++) ++ cur->frame[j]->regs[i].live = REG_LIVE_NONE; ++ } ++ ++ /* all stack frames are accessible from callee, clear them all */ ++ for (j = 0; j <= cur->curframe; j++) { ++ struct bpf_func_state *frame = cur->frame[j]; ++ struct bpf_func_state *newframe = new->frame[j]; ++ ++ for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { ++ frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; ++ frame->stack[i].spilled_ptr.parent = ++ &newframe->stack[i].spilled_ptr; ++ } ++ } + return 0; + } + +-static int do_check(struct verifier_env *env) ++/* Return true if it's OK to have the same insn return a different type. */ ++static bool reg_type_mismatch_ok(enum bpf_reg_type type) + { +- struct verifier_state *state = &env->cur_state; ++ switch (type) { ++ case PTR_TO_CTX: ++ case PTR_TO_SOCKET: ++ case PTR_TO_SOCKET_OR_NULL: ++ case PTR_TO_SOCK_COMMON: ++ case PTR_TO_SOCK_COMMON_OR_NULL: ++ case PTR_TO_TCP_SOCK: ++ case PTR_TO_TCP_SOCK_OR_NULL: ++ case PTR_TO_XDP_SOCK: ++ return false; ++ default: ++ return true; ++ } ++} ++ ++/* If an instruction was previously used with particular pointer types, then we ++ * need to be careful to avoid cases such as the below, where it may be ok ++ * for one branch accessing the pointer, but not ok for the other branch: ++ * ++ * R1 = sock_ptr ++ * goto X; ++ * ... ++ * R1 = some_other_valid_ptr; ++ * goto X; ++ * ... ++ * R2 = *(u32 *)(R1 + 0); ++ */ ++static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) ++{ ++ return src != prev && (!reg_type_mismatch_ok(src) || ++ !reg_type_mismatch_ok(prev)); ++} ++ ++static int do_check(struct bpf_verifier_env *env) ++{ ++ struct bpf_verifier_state *state; + struct bpf_insn *insns = env->prog->insnsi; +- struct reg_state *regs = state->regs; ++ struct bpf_reg_state *regs; + int insn_cnt = env->prog->len; +- int insn_idx, prev_insn_idx = 0; +- int insn_processed = 0; + bool do_print_state = false; ++ int prev_insn_idx = -1; ++ ++ env->prev_linfo = NULL; ++ ++ state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); ++ if (!state) ++ return -ENOMEM; ++ state->curframe = 0; ++ state->speculative = false; ++ state->branches = 1; ++ state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); ++ if (!state->frame[0]) { ++ kfree(state); ++ return -ENOMEM; ++ } ++ env->cur_state = state; ++ init_func_state(env, state->frame[0], ++ BPF_MAIN_FUNC /* callsite */, ++ 0 /* frameno */, ++ 0 /* subprogno, zero == main subprog */); + +- init_reg_state(regs); +- insn_idx = 0; + for (;;) { + struct bpf_insn *insn; + u8 class; + int err; + +- if (insn_idx >= insn_cnt) { +- verbose("invalid insn idx %d insn_cnt %d\n", +- insn_idx, insn_cnt); ++ env->prev_insn_idx = prev_insn_idx; ++ if (env->insn_idx >= insn_cnt) { ++ verbose(env, "invalid insn idx %d insn_cnt %d\n", ++ env->insn_idx, insn_cnt); + return -EFAULT; + } + +- insn = &insns[insn_idx]; ++ insn = &insns[env->insn_idx]; + class = BPF_CLASS(insn->code); + +- if (++insn_processed > 32768) { +- verbose("BPF program is too large. Proccessed %d insn\n", +- insn_processed); ++ if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { ++ verbose(env, ++ "BPF program is too large. Processed %d insn\n", ++ env->insn_processed); + return -E2BIG; + } + +- err = is_state_visited(env, insn_idx); ++ err = is_state_visited(env, env->insn_idx); + if (err < 0) + return err; + if (err == 1) { + /* found equivalent state, can prune the search */ +- if (log_level) { ++ if (env->log.level & BPF_LOG_LEVEL) { + if (do_print_state) +- verbose("\nfrom %d to %d: safe\n", +- prev_insn_idx, insn_idx); ++ verbose(env, "\nfrom %d to %d%s: safe\n", ++ env->prev_insn_idx, env->insn_idx, ++ env->cur_state->speculative ? ++ " (speculative execution)" : ""); + else +- verbose("%d: safe\n", insn_idx); ++ verbose(env, "%d: safe\n", env->insn_idx); + } + goto process_bpf_exit; + } + +- if (log_level && do_print_state) { +- verbose("\nfrom %d to %d:", prev_insn_idx, insn_idx); +- print_verifier_state(env); ++ if (signal_pending(current)) ++ return -EAGAIN; ++ ++ if (need_resched()) ++ cond_resched(); ++ ++ if (env->log.level & BPF_LOG_LEVEL2 || ++ (env->log.level & BPF_LOG_LEVEL && do_print_state)) { ++ if (env->log.level & BPF_LOG_LEVEL2) ++ verbose(env, "%d:", env->insn_idx); ++ else ++ verbose(env, "\nfrom %d to %d%s:", ++ env->prev_insn_idx, env->insn_idx, ++ env->cur_state->speculative ? ++ " (speculative execution)" : ""); ++ print_verifier_state(env, state->frame[state->curframe]); + do_print_state = false; + } + +- if (log_level) { +- verbose("%d: ", insn_idx); +- print_bpf_insn(env, insn); ++ if (env->log.level & BPF_LOG_LEVEL) { ++ const struct bpf_insn_cbs cbs = { ++ .cb_print = verbose, ++ .private_data = env, ++ }; ++ ++ verbose_linfo(env, env->insn_idx, "; "); ++ verbose(env, "%d: ", env->insn_idx); ++ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + } + ++ regs = cur_regs(env); ++ sanitize_mark_insn_seen(env); ++ prev_insn_idx = env->insn_idx; ++ + if (class == BPF_ALU || class == BPF_ALU64) { + err = check_alu_op(env, insn); + if (err) + return err; + + } else if (class == BPF_LDX) { +- enum bpf_reg_type src_reg_type; ++ enum bpf_reg_type *prev_src_type, src_reg_type; + + /* check for reserved fields is already done */ + + /* check src operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + +- err = check_reg_arg(regs, insn->dst_reg, DST_OP_NO_MARK); ++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); + if (err) + return err; + +@@ -1796,27 +7850,22 @@ static int do_check(struct verifier_env + /* check that memory (src_reg + off) is readable, + * the state of dst_reg will be updated by this func + */ +- err = check_mem_access(env, insn->src_reg, insn->off, +- BPF_SIZE(insn->code), BPF_READ, +- insn->dst_reg); ++ err = check_mem_access(env, env->insn_idx, insn->src_reg, ++ insn->off, BPF_SIZE(insn->code), ++ BPF_READ, insn->dst_reg, false); + if (err) + return err; + +- if (BPF_SIZE(insn->code) != BPF_W) { +- insn_idx++; +- continue; +- } ++ prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; + +- if (insn->imm == 0) { ++ if (*prev_src_type == NOT_INIT) { + /* saw a valid insn + * dst_reg = *(u32 *)(src_reg + off) +- * use reserved 'imm' field to mark this insn ++ * save type to validate intersecting paths + */ +- insn->imm = src_reg_type; ++ *prev_src_type = src_reg_type; + +- } else if (src_reg_type != insn->imm && +- (src_reg_type == PTR_TO_CTX || +- insn->imm == PTR_TO_CTX)) { ++ } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { + /* ABuser program is trying to use the same insn + * dst_reg = *(u32*) (src_reg + off) + * with different pointer types: +@@ -1824,79 +7873,98 @@ static int do_check(struct verifier_env + * src_reg == stack|map in some other branch. + * Reject it. + */ +- verbose("same insn cannot be used with different pointers\n"); ++ verbose(env, "same insn cannot be used with different pointers\n"); + return -EINVAL; + } + + } else if (class == BPF_STX) { +- enum bpf_reg_type dst_reg_type; ++ enum bpf_reg_type *prev_dst_type, dst_reg_type; + + if (BPF_MODE(insn->code) == BPF_XADD) { +- err = check_xadd(env, insn); ++ err = check_xadd(env, env->insn_idx, insn); + if (err) + return err; +- insn_idx++; ++ env->insn_idx++; + continue; + } + + /* check src1 operand */ +- err = check_reg_arg(regs, insn->src_reg, SRC_OP); ++ err = check_reg_arg(env, insn->src_reg, SRC_OP); + if (err) + return err; + /* check src2 operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + + dst_reg_type = regs[insn->dst_reg].type; + + /* check that memory (dst_reg + off) is writeable */ +- err = check_mem_access(env, insn->dst_reg, insn->off, +- BPF_SIZE(insn->code), BPF_WRITE, +- insn->src_reg); ++ err = check_mem_access(env, env->insn_idx, insn->dst_reg, ++ insn->off, BPF_SIZE(insn->code), ++ BPF_WRITE, insn->src_reg, false); + if (err) + return err; + +- if (insn->imm == 0) { +- insn->imm = dst_reg_type; +- } else if (dst_reg_type != insn->imm && +- (dst_reg_type == PTR_TO_CTX || +- insn->imm == PTR_TO_CTX)) { +- verbose("same insn cannot be used with different pointers\n"); ++ prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; ++ ++ if (*prev_dst_type == NOT_INIT) { ++ *prev_dst_type = dst_reg_type; ++ } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { ++ verbose(env, "same insn cannot be used with different pointers\n"); + return -EINVAL; + } + + } else if (class == BPF_ST) { + if (BPF_MODE(insn->code) != BPF_MEM || + insn->src_reg != BPF_REG_0) { +- verbose("BPF_ST uses reserved fields\n"); ++ verbose(env, "BPF_ST uses reserved fields\n"); + return -EINVAL; + } + /* check src operand */ +- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); ++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); + if (err) + return err; + ++ if (is_ctx_reg(env, insn->dst_reg)) { ++ verbose(env, "BPF_ST stores into R%d %s is not allowed\n", ++ insn->dst_reg, ++ reg_type_str[reg_state(env, insn->dst_reg)->type]); ++ return -EACCES; ++ } ++ + /* check that memory (dst_reg + off) is writeable */ +- err = check_mem_access(env, insn->dst_reg, insn->off, +- BPF_SIZE(insn->code), BPF_WRITE, +- -1); ++ err = check_mem_access(env, env->insn_idx, insn->dst_reg, ++ insn->off, BPF_SIZE(insn->code), ++ BPF_WRITE, -1, false); + if (err) + return err; + +- } else if (class == BPF_JMP) { ++ } else if (class == BPF_JMP || class == BPF_JMP32) { + u8 opcode = BPF_OP(insn->code); + ++ env->jmps_processed++; + if (opcode == BPF_CALL) { + if (BPF_SRC(insn->code) != BPF_K || + insn->off != 0 || +- insn->src_reg != BPF_REG_0 || +- insn->dst_reg != BPF_REG_0) { +- verbose("BPF_CALL uses reserved fields\n"); ++ (insn->src_reg != BPF_REG_0 && ++ insn->src_reg != BPF_PSEUDO_CALL) || ++ insn->dst_reg != BPF_REG_0 || ++ class == BPF_JMP32) { ++ verbose(env, "BPF_CALL uses reserved fields\n"); + return -EINVAL; + } + +- err = check_call(env, insn->imm); ++ if (env->cur_state->active_spin_lock && ++ (insn->src_reg == BPF_PSEUDO_CALL || ++ insn->imm != BPF_FUNC_spin_unlock)) { ++ verbose(env, "function calls are not allowed while holding a lock\n"); ++ return -EINVAL; ++ } ++ if (insn->src_reg == BPF_PSEUDO_CALL) ++ err = check_func_call(env, insn, &env->insn_idx); ++ else ++ err = check_helper_call(env, insn->imm, env->insn_idx); + if (err) + return err; + +@@ -1904,48 +7972,75 @@ static int do_check(struct verifier_env + if (BPF_SRC(insn->code) != BPF_K || + insn->imm != 0 || + insn->src_reg != BPF_REG_0 || +- insn->dst_reg != BPF_REG_0) { +- verbose("BPF_JA uses reserved fields\n"); ++ insn->dst_reg != BPF_REG_0 || ++ class == BPF_JMP32) { ++ verbose(env, "BPF_JA uses reserved fields\n"); + return -EINVAL; + } + +- insn_idx += insn->off + 1; ++ env->insn_idx += insn->off + 1; + continue; + + } else if (opcode == BPF_EXIT) { + if (BPF_SRC(insn->code) != BPF_K || + insn->imm != 0 || + insn->src_reg != BPF_REG_0 || +- insn->dst_reg != BPF_REG_0) { +- verbose("BPF_EXIT uses reserved fields\n"); ++ insn->dst_reg != BPF_REG_0 || ++ class == BPF_JMP32) { ++ verbose(env, "BPF_EXIT uses reserved fields\n"); + return -EINVAL; + } + ++ if (env->cur_state->active_spin_lock) { ++ verbose(env, "bpf_spin_unlock is missing\n"); ++ return -EINVAL; ++ } ++ ++ if (state->curframe) { ++ /* exit from nested function */ ++ err = prepare_func_exit(env, &env->insn_idx); ++ if (err) ++ return err; ++ do_print_state = true; ++ continue; ++ } ++ ++ err = check_reference_leak(env); ++ if (err) ++ return err; ++ + /* eBPF calling convetion is such that R0 is used + * to return the value from eBPF program. + * Make sure that it's readable at this time + * of bpf_exit, which means that program wrote + * something into it earlier + */ +- err = check_reg_arg(regs, BPF_REG_0, SRC_OP); ++ err = check_reg_arg(env, BPF_REG_0, SRC_OP); + if (err) + return err; + + if (is_pointer_value(env, BPF_REG_0)) { +- verbose("R0 leaks addr as return value\n"); ++ verbose(env, "R0 leaks addr as return value\n"); + return -EACCES; + } + ++ err = check_return_code(env); ++ if (err) ++ return err; + process_bpf_exit: +- insn_idx = pop_stack(env, &prev_insn_idx); +- if (insn_idx < 0) { ++ update_branch_counts(env, env->cur_state); ++ err = pop_stack(env, &prev_insn_idx, ++ &env->insn_idx); ++ if (err < 0) { ++ if (err != -ENOENT) ++ return err; + break; + } else { + do_print_state = true; + continue; + } + } else { +- err = check_cond_jmp_op(env, insn, &insn_idx); ++ err = check_cond_jmp_op(env, insn, &env->insn_idx); + if (err) + return err; + } +@@ -1962,83 +8057,194 @@ process_bpf_exit: + if (err) + return err; + +- insn_idx++; ++ env->insn_idx++; ++ sanitize_mark_insn_seen(env); + } else { +- verbose("invalid BPF_LD mode\n"); ++ verbose(env, "invalid BPF_LD mode\n"); + return -EINVAL; + } + } else { +- verbose("unknown insn class %d\n", class); ++ verbose(env, "unknown insn class %d\n", class); + return -EINVAL; + } + +- insn_idx++; ++ env->insn_idx++; + } + ++ env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; + return 0; + } + ++static int check_map_prealloc(struct bpf_map *map) ++{ ++ return (map->map_type != BPF_MAP_TYPE_HASH && ++ map->map_type != BPF_MAP_TYPE_PERCPU_HASH && ++ map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || ++ !(map->map_flags & BPF_F_NO_PREALLOC); ++} ++ ++static bool is_tracing_prog_type(enum bpf_prog_type type) ++{ ++ switch (type) { ++ case BPF_PROG_TYPE_KPROBE: ++ case BPF_PROG_TYPE_TRACEPOINT: ++ case BPF_PROG_TYPE_PERF_EVENT: ++ case BPF_PROG_TYPE_RAW_TRACEPOINT: ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static int check_map_prog_compatibility(struct bpf_verifier_env *env, ++ struct bpf_map *map, ++ struct bpf_prog *prog) ++ ++{ ++ /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use ++ * preallocated hash maps, since doing memory allocation ++ * in overflow_handler can crash depending on where nmi got ++ * triggered. ++ */ ++ if (prog->type == BPF_PROG_TYPE_PERF_EVENT) { ++ if (!check_map_prealloc(map)) { ++ verbose(env, "perf_event programs can only use preallocated hash map\n"); ++ return -EINVAL; ++ } ++ if (map->inner_map_meta && ++ !check_map_prealloc(map->inner_map_meta)) { ++ verbose(env, "perf_event programs can only use preallocated inner hash map\n"); ++ return -EINVAL; ++ } ++ } ++ ++ if ((is_tracing_prog_type(prog->type) || ++ prog->type == BPF_PROG_TYPE_SOCKET_FILTER) && ++ map_value_has_spin_lock(map)) { ++ verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++static bool bpf_map_is_cgroup_storage(struct bpf_map *map) ++{ ++ return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || ++ map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); ++} ++ + /* look for pseudo eBPF instructions that access map FDs and + * replace them with actual map pointers + */ +-static int replace_map_fd_with_map_ptr(struct verifier_env *env) ++static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) + { + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; +- int i, j; ++ int i, j, err; ++ ++ err = bpf_prog_calc_tag(env->prog); ++ if (err) ++ return err; + + for (i = 0; i < insn_cnt; i++, insn++) { + if (BPF_CLASS(insn->code) == BPF_LDX && + (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { +- verbose("BPF_LDX uses reserved fields\n"); ++ verbose(env, "BPF_LDX uses reserved fields\n"); + return -EINVAL; + } + + if (BPF_CLASS(insn->code) == BPF_STX && + ((BPF_MODE(insn->code) != BPF_MEM && + BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { +- verbose("BPF_STX uses reserved fields\n"); ++ verbose(env, "BPF_STX uses reserved fields\n"); + return -EINVAL; + } + + if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { ++ struct bpf_insn_aux_data *aux; + struct bpf_map *map; + struct fd f; ++ u64 addr; + + if (i == insn_cnt - 1 || insn[1].code != 0 || + insn[1].dst_reg != 0 || insn[1].src_reg != 0 || + insn[1].off != 0) { +- verbose("invalid bpf_ld_imm64 insn\n"); ++ verbose(env, "invalid bpf_ld_imm64 insn\n"); + return -EINVAL; + } + +- if (insn->src_reg == 0) ++ if (insn[0].src_reg == 0) + /* valid generic load 64-bit imm */ + goto next_insn; + +- if (insn->src_reg != BPF_PSEUDO_MAP_FD) { +- verbose("unrecognized bpf_ld_imm64 insn\n"); ++ /* In final convert_pseudo_ld_imm64() step, this is ++ * converted into regular 64-bit imm load insn. ++ */ ++ if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && ++ insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || ++ (insn[0].src_reg == BPF_PSEUDO_MAP_FD && ++ insn[1].imm != 0)) { ++ verbose(env, ++ "unrecognized bpf_ld_imm64 insn\n"); + return -EINVAL; + } + +- f = fdget(insn->imm); ++ f = fdget(insn[0].imm); + map = __bpf_map_get(f); + if (IS_ERR(map)) { +- verbose("fd %d is not pointing to valid bpf_map\n", +- insn->imm); ++ verbose(env, "fd %d is not pointing to valid bpf_map\n", ++ insn[0].imm); + return PTR_ERR(map); + } + +- /* store map pointer inside BPF_LD_IMM64 instruction */ +- insn[0].imm = (u32) (unsigned long) map; +- insn[1].imm = ((u64) (unsigned long) map) >> 32; ++ err = check_map_prog_compatibility(env, map, env->prog); ++ if (err) { ++ fdput(f); ++ return err; ++ } ++ ++ aux = &env->insn_aux_data[i]; ++ if (insn->src_reg == BPF_PSEUDO_MAP_FD) { ++ addr = (unsigned long)map; ++ } else { ++ u32 off = insn[1].imm; ++ ++ if (off >= BPF_MAX_VAR_OFF) { ++ verbose(env, "direct value offset of %u is not allowed\n", off); ++ fdput(f); ++ return -EINVAL; ++ } ++ ++ if (!map->ops->map_direct_value_addr) { ++ verbose(env, "no direct value access support for this map type\n"); ++ fdput(f); ++ return -EINVAL; ++ } ++ ++ err = map->ops->map_direct_value_addr(map, &addr, off); ++ if (err) { ++ verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", ++ map->value_size, off); ++ fdput(f); ++ return err; ++ } ++ ++ aux->map_off = off; ++ addr += off; ++ } ++ ++ insn[0].imm = (u32)addr; ++ insn[1].imm = addr >> 32; + + /* check whether we recorded this map already */ +- for (j = 0; j < env->used_map_cnt; j++) ++ for (j = 0; j < env->used_map_cnt; j++) { + if (env->used_maps[j] == map) { ++ aux->map_index = j; + fdput(f); + goto next_insn; + } ++ } + + if (env->used_map_cnt >= MAX_USED_MAPS) { + fdput(f); +@@ -2048,19 +8254,31 @@ static int replace_map_fd_with_map_ptr(s + /* hold the map. If the program is rejected by verifier, + * the map will be released by release_maps() or it + * will be used by the valid program until it's unloaded +- * and all maps are released in free_bpf_prog_info() ++ * and all maps are released in free_used_maps() + */ + map = bpf_map_inc(map, false); + if (IS_ERR(map)) { + fdput(f); + return PTR_ERR(map); + } ++ ++ aux->map_index = env->used_map_cnt; + env->used_maps[env->used_map_cnt++] = map; + ++ if (bpf_map_is_cgroup_storage(map)) ++ return -EINVAL; ++ + fdput(f); + next_insn: + insn++; + i++; ++ continue; ++ } ++ ++ /* Basic sanity check before we invest more work here. */ ++ if (!bpf_opcode_in_insntable(insn->code)) { ++ verbose(env, "unknown opcode %02x\n", insn->code); ++ return -EINVAL; + } + } + +@@ -2072,7 +8290,7 @@ next_insn: + } + + /* drop refcnt of maps used by the rejected program */ +-static void release_maps(struct verifier_env *env) ++static void release_maps(struct bpf_verifier_env *env) + { + int i; + +@@ -2081,7 +8299,7 @@ static void release_maps(struct verifier + } + + /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ +-static void convert_pseudo_ld_imm64(struct verifier_env *env) ++static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) + { + struct bpf_insn *insn = env->prog->insnsi; + int insn_cnt = env->prog->len; +@@ -2092,201 +8310,1266 @@ static void convert_pseudo_ld_imm64(stru + insn->src_reg = 0; + } + +-static void adjust_branches(struct bpf_prog *prog, int pos, int delta) ++/* single env->prog->insni[off] instruction was replaced with the range ++ * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying ++ * [0, off) and [off, end) to new locations, so the patched range stays zero ++ */ ++static int adjust_insn_aux_data(struct bpf_verifier_env *env, ++ struct bpf_prog *new_prog, u32 off, u32 cnt) + { +- struct bpf_insn *insn = prog->insnsi; +- int insn_cnt = prog->len; ++ struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; ++ struct bpf_insn *insn = new_prog->insnsi; ++ bool old_seen = old_data[off].seen; ++ u32 prog_len; + int i; + +- for (i = 0; i < insn_cnt; i++, insn++) { +- if (BPF_CLASS(insn->code) != BPF_JMP || +- BPF_OP(insn->code) == BPF_CALL || +- BPF_OP(insn->code) == BPF_EXIT) ++ /* aux info at OFF always needs adjustment, no matter fast path ++ * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the ++ * original insn at old prog. ++ */ ++ old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); ++ ++ if (cnt == 1) ++ return 0; ++ prog_len = new_prog->len; ++ new_data = vzalloc(array_size(prog_len, ++ sizeof(struct bpf_insn_aux_data))); ++ if (!new_data) ++ return -ENOMEM; ++ memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); ++ memcpy(new_data + off + cnt - 1, old_data + off, ++ sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); ++ for (i = off; i < off + cnt - 1; i++) { ++ /* Expand insni[off]'s seen count to the patched range. */ ++ new_data[i].seen = old_seen; ++ new_data[i].zext_dst = insn_has_def32(env, insn + i); ++ } ++ env->insn_aux_data = new_data; ++ vfree(old_data); ++ return 0; ++} ++ ++static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) ++{ ++ int i; ++ ++ if (len == 1) ++ return; ++ /* NOTE: fake 'exit' subprog should be updated as well. */ ++ for (i = 0; i <= env->subprog_cnt; i++) { ++ if (env->subprog_info[i].start <= off) + continue; ++ env->subprog_info[i].start += len - 1; ++ } ++} ++ ++static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, ++ const struct bpf_insn *patch, u32 len) ++{ ++ struct bpf_prog *new_prog; ++ ++ new_prog = bpf_patch_insn_single(env->prog, off, patch, len); ++ if (IS_ERR(new_prog)) { ++ if (PTR_ERR(new_prog) == -ERANGE) ++ verbose(env, ++ "insn %d cannot be patched due to 16-bit range\n", ++ env->insn_aux_data[off].orig_idx); ++ return NULL; ++ } ++ if (adjust_insn_aux_data(env, new_prog, off, len)) ++ return NULL; ++ adjust_subprog_starts(env, off, len); ++ return new_prog; ++} + +- /* adjust offset of jmps if necessary */ +- if (i < pos && i + insn->off + 1 > pos) +- insn->off += delta; +- else if (i > pos + delta && i + insn->off + 1 <= pos + delta) +- insn->off -= delta; ++static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, ++ u32 off, u32 cnt) ++{ ++ int i, j; ++ ++ /* find first prog starting at or after off (first to remove) */ ++ for (i = 0; i < env->subprog_cnt; i++) ++ if (env->subprog_info[i].start >= off) ++ break; ++ /* find first prog starting at or after off + cnt (first to stay) */ ++ for (j = i; j < env->subprog_cnt; j++) ++ if (env->subprog_info[j].start >= off + cnt) ++ break; ++ /* if j doesn't start exactly at off + cnt, we are just removing ++ * the front of previous prog ++ */ ++ if (env->subprog_info[j].start != off + cnt) ++ j--; ++ ++ if (j > i) { ++ struct bpf_prog_aux *aux = env->prog->aux; ++ int move; ++ ++ /* move fake 'exit' subprog as well */ ++ move = env->subprog_cnt + 1 - j; ++ ++ memmove(env->subprog_info + i, ++ env->subprog_info + j, ++ sizeof(*env->subprog_info) * move); ++ env->subprog_cnt -= j - i; ++ ++ /* remove func_info */ ++ if (aux->func_info) { ++ move = aux->func_info_cnt - j; ++ ++ memmove(aux->func_info + i, ++ aux->func_info + j, ++ sizeof(*aux->func_info) * move); ++ aux->func_info_cnt -= j - i; ++ /* func_info->insn_off is set after all code rewrites, ++ * in adjust_btf_func() - no need to adjust ++ */ ++ } ++ } else { ++ /* convert i from "first prog to remove" to "first to adjust" */ ++ if (env->subprog_info[i].start == off) ++ i++; + } ++ ++ /* update fake 'exit' subprog as well */ ++ for (; i <= env->subprog_cnt; i++) ++ env->subprog_info[i].start -= cnt; ++ ++ return 0; ++} ++ ++static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, ++ u32 cnt) ++{ ++ struct bpf_prog *prog = env->prog; ++ u32 i, l_off, l_cnt, nr_linfo; ++ struct bpf_line_info *linfo; ++ ++ nr_linfo = prog->aux->nr_linfo; ++ if (!nr_linfo) ++ return 0; ++ ++ linfo = prog->aux->linfo; ++ ++ /* find first line info to remove, count lines to be removed */ ++ for (i = 0; i < nr_linfo; i++) ++ if (linfo[i].insn_off >= off) ++ break; ++ ++ l_off = i; ++ l_cnt = 0; ++ for (; i < nr_linfo; i++) ++ if (linfo[i].insn_off < off + cnt) ++ l_cnt++; ++ else ++ break; ++ ++ /* First live insn doesn't match first live linfo, it needs to "inherit" ++ * last removed linfo. prog is already modified, so prog->len == off ++ * means no live instructions after (tail of the program was removed). ++ */ ++ if (prog->len != off && l_cnt && ++ (i == nr_linfo || linfo[i].insn_off != off + cnt)) { ++ l_cnt--; ++ linfo[--i].insn_off = off + cnt; ++ } ++ ++ /* remove the line info which refer to the removed instructions */ ++ if (l_cnt) { ++ memmove(linfo + l_off, linfo + i, ++ sizeof(*linfo) * (nr_linfo - i)); ++ ++ prog->aux->nr_linfo -= l_cnt; ++ nr_linfo = prog->aux->nr_linfo; ++ } ++ ++ /* pull all linfo[i].insn_off >= off + cnt in by cnt */ ++ for (i = l_off; i < nr_linfo; i++) ++ linfo[i].insn_off -= cnt; ++ ++ /* fix up all subprogs (incl. 'exit') which start >= off */ ++ for (i = 0; i <= env->subprog_cnt; i++) ++ if (env->subprog_info[i].linfo_idx > l_off) { ++ /* program may have started in the removed region but ++ * may not be fully removed ++ */ ++ if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) ++ env->subprog_info[i].linfo_idx -= l_cnt; ++ else ++ env->subprog_info[i].linfo_idx = l_off; ++ } ++ ++ return 0; ++} ++ ++static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) ++{ ++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; ++ unsigned int orig_prog_len = env->prog->len; ++ int err; ++ ++ err = bpf_remove_insns(env->prog, off, cnt); ++ if (err) ++ return err; ++ ++ err = adjust_subprog_starts_after_remove(env, off, cnt); ++ if (err) ++ return err; ++ ++ err = bpf_adj_linfo_after_remove(env, off, cnt); ++ if (err) ++ return err; ++ ++ memmove(aux_data + off, aux_data + off + cnt, ++ sizeof(*aux_data) * (orig_prog_len - off - cnt)); ++ ++ return 0; + } + +-/* convert load instructions that access fields of 'struct __sk_buff' +- * into sequence of instructions that access fields of 'struct sk_buff' ++/* The verifier does more data flow analysis than llvm and will not ++ * explore branches that are dead at run time. Malicious programs can ++ * have dead code too. Therefore replace all dead at-run-time code ++ * with 'ja -1'. ++ * ++ * Just nops are not optimal, e.g. if they would sit at the end of the ++ * program and through another bug we would manage to jump there, then ++ * we'd execute beyond program memory otherwise. Returning exception ++ * code also wouldn't work since we can have subprogs where the dead ++ * code could be located. + */ +-static int convert_ctx_accesses(struct verifier_env *env) ++static void sanitize_dead_code(struct bpf_verifier_env *env) + { ++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; ++ struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); + struct bpf_insn *insn = env->prog->insnsi; ++ const int insn_cnt = env->prog->len; ++ int i; ++ ++ for (i = 0; i < insn_cnt; i++) { ++ if (aux_data[i].seen) ++ continue; ++ memcpy(insn + i, &trap, sizeof(trap)); ++ } ++} ++ ++static bool insn_is_cond_jump(u8 code) ++{ ++ u8 op; ++ ++ if (BPF_CLASS(code) == BPF_JMP32) ++ return true; ++ ++ if (BPF_CLASS(code) != BPF_JMP) ++ return false; ++ ++ op = BPF_OP(code); ++ return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; ++} ++ ++static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) ++{ ++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; ++ struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); ++ struct bpf_insn *insn = env->prog->insnsi; ++ const int insn_cnt = env->prog->len; ++ int i; ++ ++ for (i = 0; i < insn_cnt; i++, insn++) { ++ if (!insn_is_cond_jump(insn->code)) ++ continue; ++ ++ if (!aux_data[i + 1].seen) ++ ja.off = insn->off; ++ else if (!aux_data[i + 1 + insn->off].seen) ++ ja.off = 0; ++ else ++ continue; ++ ++ memcpy(insn, &ja, sizeof(ja)); ++ } ++} ++ ++static int opt_remove_dead_code(struct bpf_verifier_env *env) ++{ ++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; + int insn_cnt = env->prog->len; +- struct bpf_insn insn_buf[16]; ++ int i, err; ++ ++ for (i = 0; i < insn_cnt; i++) { ++ int j; ++ ++ j = 0; ++ while (i + j < insn_cnt && !aux_data[i + j].seen) ++ j++; ++ if (!j) ++ continue; ++ ++ err = verifier_remove_insns(env, i, j); ++ if (err) ++ return err; ++ insn_cnt = env->prog->len; ++ } ++ ++ return 0; ++} ++ ++static int opt_remove_nops(struct bpf_verifier_env *env) ++{ ++ const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); ++ struct bpf_insn *insn = env->prog->insnsi; ++ int insn_cnt = env->prog->len; ++ int i, err; ++ ++ for (i = 0; i < insn_cnt; i++) { ++ if (memcmp(&insn[i], &ja, sizeof(ja))) ++ continue; ++ ++ err = verifier_remove_insns(env, i, 1); ++ if (err) ++ return err; ++ insn_cnt--; ++ i--; ++ } ++ ++ return 0; ++} ++ ++static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, ++ const union bpf_attr *attr) ++{ ++ struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; ++ struct bpf_insn_aux_data *aux = env->insn_aux_data; ++ int i, patch_len, delta = 0, len = env->prog->len; ++ struct bpf_insn *insns = env->prog->insnsi; ++ struct bpf_prog *new_prog; ++ bool rnd_hi32; ++ ++ rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; ++ zext_patch[1] = BPF_ZEXT_REG(0); ++ rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); ++ rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); ++ rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); ++ for (i = 0; i < len; i++) { ++ int adj_idx = i + delta; ++ struct bpf_insn insn; ++ ++ insn = insns[adj_idx]; ++ if (!aux[adj_idx].zext_dst) { ++ u8 code, class; ++ u32 imm_rnd; ++ ++ if (!rnd_hi32) ++ continue; ++ ++ code = insn.code; ++ class = BPF_CLASS(code); ++ if (insn_no_def(&insn)) ++ continue; ++ ++ /* NOTE: arg "reg" (the fourth one) is only used for ++ * BPF_STX which has been ruled out in above ++ * check, it is safe to pass NULL here. ++ */ ++ if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) { ++ if (class == BPF_LD && ++ BPF_MODE(code) == BPF_IMM) ++ i++; ++ continue; ++ } ++ ++ /* ctx load could be transformed into wider load. */ ++ if (class == BPF_LDX && ++ aux[adj_idx].ptr_type == PTR_TO_CTX) ++ continue; ++ ++ imm_rnd = get_random_int(); ++ rnd_hi32_patch[0] = insn; ++ rnd_hi32_patch[1].imm = imm_rnd; ++ rnd_hi32_patch[3].dst_reg = insn.dst_reg; ++ patch = rnd_hi32_patch; ++ patch_len = 4; ++ goto apply_patch_buffer; ++ } ++ ++ if (!bpf_jit_needs_zext()) ++ continue; ++ ++ zext_patch[0] = insn; ++ zext_patch[1].dst_reg = insn.dst_reg; ++ zext_patch[1].src_reg = insn.dst_reg; ++ patch = zext_patch; ++ patch_len = 2; ++apply_patch_buffer: ++ new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); ++ if (!new_prog) ++ return -ENOMEM; ++ env->prog = new_prog; ++ insns = new_prog->insnsi; ++ aux = env->insn_aux_data; ++ delta += patch_len - 1; ++ } ++ ++ return 0; ++} ++ ++/* convert load instructions that access fields of a context type into a ++ * sequence of instructions that access fields of the underlying structure: ++ * struct __sk_buff -> struct sk_buff ++ * struct bpf_sock_ops -> struct sock ++ */ ++static int convert_ctx_accesses(struct bpf_verifier_env *env) ++{ ++ const struct bpf_verifier_ops *ops = env->ops; ++ int i, cnt, size, ctx_field_size, delta = 0; ++ const int insn_cnt = env->prog->len; ++ struct bpf_insn insn_buf[16], *insn; ++ u32 target_size, size_default, off; + struct bpf_prog *new_prog; +- u32 cnt; +- int i; + enum bpf_access_type type; ++ bool is_narrower_load; ++ ++ if (ops->gen_prologue || env->seen_direct_write) { ++ if (!ops->gen_prologue) { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } ++ cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, ++ env->prog); ++ if (cnt >= ARRAY_SIZE(insn_buf)) { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } else if (cnt) { ++ new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ env->prog = new_prog; ++ delta += cnt - 1; ++ } ++ } + +- if (!env->prog->aux->ops->convert_ctx_access) ++ if (bpf_prog_is_dev_bound(env->prog->aux)) + return 0; + ++ insn = env->prog->insnsi + delta; ++ + for (i = 0; i < insn_cnt; i++, insn++) { +- if (insn->code == (BPF_LDX | BPF_MEM | BPF_W)) ++ bpf_convert_ctx_access_t convert_ctx_access; ++ ++ if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || ++ insn->code == (BPF_LDX | BPF_MEM | BPF_H) || ++ insn->code == (BPF_LDX | BPF_MEM | BPF_W) || ++ insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) + type = BPF_READ; +- else if (insn->code == (BPF_STX | BPF_MEM | BPF_W)) ++ else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || ++ insn->code == (BPF_STX | BPF_MEM | BPF_H) || ++ insn->code == (BPF_STX | BPF_MEM | BPF_W) || ++ insn->code == (BPF_STX | BPF_MEM | BPF_DW)) + type = BPF_WRITE; + else + continue; + +- if (insn->imm != PTR_TO_CTX) { +- /* clear internal mark */ +- insn->imm = 0; ++ if (type == BPF_WRITE && ++ env->insn_aux_data[i + delta].sanitize_stack_off) { ++ struct bpf_insn patch[] = { ++ /* Sanitize suspicious stack slot with zero. ++ * There are no memory dependencies for this store, ++ * since it's only using frame pointer and immediate ++ * constant of zero ++ */ ++ BPF_ST_MEM(BPF_DW, BPF_REG_FP, ++ env->insn_aux_data[i + delta].sanitize_stack_off, ++ 0), ++ /* the original STX instruction will immediately ++ * overwrite the same stack slot with appropriate value ++ */ ++ *insn, ++ }; ++ ++ cnt = ARRAY_SIZE(patch); ++ new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ switch (env->insn_aux_data[i + delta].ptr_type) { ++ case PTR_TO_CTX: ++ if (!ops->convert_ctx_access) ++ continue; ++ convert_ctx_access = ops->convert_ctx_access; ++ break; ++ default: + continue; + } + +- cnt = env->prog->aux->ops-> +- convert_ctx_access(type, insn->dst_reg, insn->src_reg, +- insn->off, insn_buf, env->prog); +- if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { +- verbose("bpf verifier is misconfigured\n"); ++ ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; ++ size = BPF_LDST_BYTES(insn); ++ ++ /* If the read access is a narrower load of the field, ++ * convert to a 4/8-byte load, to minimum program type specific ++ * convert_ctx_access changes. If conversion is successful, ++ * we will apply proper mask to the result. ++ */ ++ is_narrower_load = size < ctx_field_size; ++ size_default = bpf_ctx_off_adjust_machine(ctx_field_size); ++ off = insn->off; ++ if (is_narrower_load) { ++ u8 size_code; ++ ++ if (type == BPF_WRITE) { ++ verbose(env, "bpf verifier narrow ctx access misconfigured\n"); ++ return -EINVAL; ++ } ++ ++ size_code = BPF_H; ++ if (ctx_field_size == 4) ++ size_code = BPF_W; ++ else if (ctx_field_size == 8) ++ size_code = BPF_DW; ++ ++ insn->off = off & ~(size_default - 1); ++ insn->code = BPF_LDX | BPF_MEM | size_code; ++ } ++ ++ target_size = 0; ++ cnt = convert_ctx_access(type, insn, insn_buf, env->prog, ++ &target_size); ++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || ++ (ctx_field_size && !target_size)) { ++ verbose(env, "bpf verifier is misconfigured\n"); + return -EINVAL; + } + +- if (cnt == 1) { +- memcpy(insn, insn_buf, sizeof(*insn)); +- continue; ++ if (is_narrower_load && size < target_size) { ++ u8 shift = bpf_ctx_narrow_access_offset( ++ off, size, size_default) * 8; ++ if (ctx_field_size <= 4) { ++ if (shift) ++ insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, ++ insn->dst_reg, ++ shift); ++ insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, ++ (1 << size * 8) - 1); ++ } else { ++ if (shift) ++ insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, ++ insn->dst_reg, ++ shift); ++ insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, ++ (1ULL << size * 8) - 1); ++ } + } + +- /* several new insns need to be inserted. Make room for them */ +- insn_cnt += cnt - 1; +- new_prog = bpf_prog_realloc(env->prog, +- bpf_prog_size(insn_cnt), +- GFP_USER); ++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + +- new_prog->len = insn_cnt; ++ delta += cnt - 1; + +- memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1, +- sizeof(*insn) * (insn_cnt - i - cnt)); ++ /* keep walking new program and skip insns we just inserted */ ++ env->prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ } + +- /* copy substitute insns in place of load instruction */ +- memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt); ++ return 0; ++} + +- /* adjust branches in the whole program */ +- adjust_branches(new_prog, i, cnt - 1); ++static int jit_subprogs(struct bpf_verifier_env *env) ++{ ++ struct bpf_prog *prog = env->prog, **func, *tmp; ++ int i, j, subprog_start, subprog_end = 0, len, subprog; ++ struct bpf_insn *insn; ++ void *old_bpf_func; ++ int err; + +- /* keep walking new program and skip insns we just inserted */ +- env->prog = new_prog; +- insn = new_prog->insnsi + i + cnt - 1; +- i += cnt - 1; ++ if (env->subprog_cnt <= 1) ++ return 0; ++ ++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { ++ if (insn->code != (BPF_JMP | BPF_CALL) || ++ insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ /* Upon error here we cannot fall back to interpreter but ++ * need a hard reject of the program. Thus -EFAULT is ++ * propagated in any case. ++ */ ++ subprog = find_subprog(env, i + insn->imm + 1); ++ if (subprog < 0) { ++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", ++ i + insn->imm + 1); ++ return -EFAULT; ++ } ++ /* temporarily remember subprog id inside insn instead of ++ * aux_data, since next loop will split up all insns into funcs ++ */ ++ insn->off = subprog; ++ /* remember original imm in case JIT fails and fallback ++ * to interpreter will be needed ++ */ ++ env->insn_aux_data[i].call_imm = insn->imm; ++ /* point imm to __bpf_call_base+1 from JITs point of view */ ++ insn->imm = 1; ++ } ++ ++ err = bpf_prog_alloc_jited_linfo(prog); ++ if (err) ++ goto out_undo_insn; ++ ++ err = -ENOMEM; ++ func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); ++ if (!func) ++ goto out_undo_insn; ++ ++ for (i = 0; i < env->subprog_cnt; i++) { ++ subprog_start = subprog_end; ++ subprog_end = env->subprog_info[i + 1].start; ++ ++ len = subprog_end - subprog_start; ++ /* BPF_PROG_RUN doesn't call subprogs directly, ++ * hence main prog stats include the runtime of subprogs. ++ * subprogs don't have IDs and not reachable via prog_get_next_id ++ * func[i]->aux->stats will never be accessed and stays NULL ++ */ ++ func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); ++ if (!func[i]) ++ goto out_free; ++ memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], ++ len * sizeof(struct bpf_insn)); ++ func[i]->type = prog->type; ++ func[i]->len = len; ++ if (bpf_prog_calc_tag(func[i])) ++ goto out_free; ++ func[i]->is_func = 1; ++ func[i]->aux->func_idx = i; ++ /* the btf and func_info will be freed only at prog->aux */ ++ func[i]->aux->btf = prog->aux->btf; ++ func[i]->aux->func_info = prog->aux->func_info; ++ ++ /* Use bpf_prog_F_tag to indicate functions in stack traces. ++ * Long term would need debug info to populate names ++ */ ++ func[i]->aux->name[0] = 'F'; ++ func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; ++ func[i]->jit_requested = 1; ++ func[i]->aux->linfo = prog->aux->linfo; ++ func[i]->aux->nr_linfo = prog->aux->nr_linfo; ++ func[i]->aux->jited_linfo = prog->aux->jited_linfo; ++ func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; ++ func[i] = bpf_int_jit_compile(func[i]); ++ if (!func[i]->jited) { ++ err = -ENOTSUPP; ++ goto out_free; ++ } ++ cond_resched(); ++ } ++ /* at this point all bpf functions were successfully JITed ++ * now populate all bpf_calls with correct addresses and ++ * run last pass of JIT ++ */ ++ for (i = 0; i < env->subprog_cnt; i++) { ++ insn = func[i]->insnsi; ++ for (j = 0; j < func[i]->len; j++, insn++) { ++ if (insn->code != (BPF_JMP | BPF_CALL) || ++ insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ subprog = insn->off; ++ insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - ++ __bpf_call_base; ++ } ++ ++ /* we use the aux data to keep a list of the start addresses ++ * of the JITed images for each function in the program ++ * ++ * for some architectures, such as powerpc64, the imm field ++ * might not be large enough to hold the offset of the start ++ * address of the callee's JITed image from __bpf_call_base ++ * ++ * in such cases, we can lookup the start address of a callee ++ * by using its subprog id, available from the off field of ++ * the call instruction, as an index for this list ++ */ ++ func[i]->aux->func = func; ++ func[i]->aux->func_cnt = env->subprog_cnt; ++ } ++ for (i = 0; i < env->subprog_cnt; i++) { ++ old_bpf_func = func[i]->bpf_func; ++ tmp = bpf_int_jit_compile(func[i]); ++ if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { ++ verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); ++ err = -ENOTSUPP; ++ goto out_free; ++ } ++ cond_resched(); ++ } ++ ++ /* finally lock prog and jit images for all functions and ++ * populate kallsysm ++ */ ++ for (i = 0; i < env->subprog_cnt; i++) { ++ bpf_prog_lock_ro(func[i]); ++ bpf_prog_kallsyms_add(func[i]); ++ } ++ ++ /* Last step: make now unused interpreter insns from main ++ * prog consistent for later dump requests, so they can ++ * later look the same as if they were interpreted only. ++ */ ++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { ++ if (insn->code != (BPF_JMP | BPF_CALL) || ++ insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ insn->off = env->insn_aux_data[i].call_imm; ++ subprog = find_subprog(env, i + insn->off + 1); ++ insn->imm = subprog; + } + ++ prog->jited = 1; ++ prog->bpf_func = func[0]->bpf_func; ++ prog->aux->func = func; ++ prog->aux->func_cnt = env->subprog_cnt; ++ bpf_prog_free_unused_jited_linfo(prog); + return 0; ++out_free: ++ for (i = 0; i < env->subprog_cnt; i++) ++ if (func[i]) ++ bpf_jit_free(func[i]); ++ kfree(func); ++out_undo_insn: ++ /* cleanup main prog to be interpreted */ ++ prog->jit_requested = 0; ++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { ++ if (insn->code != (BPF_JMP | BPF_CALL) || ++ insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ insn->off = 0; ++ insn->imm = env->insn_aux_data[i].call_imm; ++ } ++ bpf_prog_free_jited_linfo(prog); ++ return err; + } + +-static void free_states(struct verifier_env *env) ++static int fixup_call_args(struct bpf_verifier_env *env) + { +- struct verifier_state_list *sl, *sln; ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++ struct bpf_prog *prog = env->prog; ++ struct bpf_insn *insn = prog->insnsi; ++ int i, depth; ++#endif ++ int err = 0; ++ ++ if (env->prog->jit_requested && ++ !bpf_prog_is_dev_bound(env->prog->aux)) { ++ err = jit_subprogs(env); ++ if (err == 0) ++ return 0; ++ if (err == -EFAULT) ++ return err; ++ } ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++ for (i = 0; i < prog->len; i++, insn++) { ++ if (insn->code != (BPF_JMP | BPF_CALL) || ++ insn->src_reg != BPF_PSEUDO_CALL) ++ continue; ++ depth = get_callee_stack_depth(env, insn, i); ++ if (depth < 0) ++ return depth; ++ bpf_patch_call_args(insn, depth); ++ } ++ err = 0; ++#endif ++ return err; ++} ++ ++/* fixup insn->imm field of bpf_call instructions ++ * and inline eligible helpers as explicit sequence of BPF instructions ++ * ++ * this function is called after eBPF program passed verification ++ */ ++static int fixup_bpf_calls(struct bpf_verifier_env *env) ++{ ++ struct bpf_prog *prog = env->prog; ++ struct bpf_insn *insn = prog->insnsi; ++ const struct bpf_func_proto *fn; ++ const int insn_cnt = prog->len; ++ const struct bpf_map_ops *ops; ++ struct bpf_insn_aux_data *aux; ++ struct bpf_insn insn_buf[16]; ++ struct bpf_prog *new_prog; ++ struct bpf_map *map_ptr; ++ int i, cnt, delta = 0; ++ ++ for (i = 0; i < insn_cnt; i++, insn++) { ++ if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || ++ insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || ++ insn->code == (BPF_ALU | BPF_MOD | BPF_X) || ++ insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { ++ bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; ++ bool isdiv = BPF_OP(insn->code) == BPF_DIV; ++ struct bpf_insn *patchlet; ++ struct bpf_insn chk_and_div[] = { ++ /* [R,W]x div 0 -> 0 */ ++ BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | ++ BPF_JNE | BPF_K, insn->src_reg, ++ 0, 2, 0), ++ BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), ++ BPF_JMP_IMM(BPF_JA, 0, 0, 1), ++ *insn, ++ }; ++ struct bpf_insn chk_and_mod[] = { ++ /* [R,W]x mod 0 -> [R,W]x */ ++ BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | ++ BPF_JEQ | BPF_K, insn->src_reg, ++ 0, 1 + (is64 ? 0 : 1), 0), ++ *insn, ++ BPF_JMP_IMM(BPF_JA, 0, 0, 1), ++ BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), ++ }; ++ ++ patchlet = isdiv ? chk_and_div : chk_and_mod; ++ cnt = isdiv ? ARRAY_SIZE(chk_and_div) : ++ ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); ++ ++ new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ if (BPF_CLASS(insn->code) == BPF_LD && ++ (BPF_MODE(insn->code) == BPF_ABS || ++ BPF_MODE(insn->code) == BPF_IND)) { ++ cnt = env->ops->gen_ld_abs(insn, insn_buf); ++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } ++ ++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || ++ insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { ++ const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; ++ const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; ++ struct bpf_insn insn_buf[16]; ++ struct bpf_insn *patch = &insn_buf[0]; ++ bool issrc, isneg, isimm; ++ u32 off_reg; ++ ++ aux = &env->insn_aux_data[i + delta]; ++ if (!aux->alu_state || ++ aux->alu_state == BPF_ALU_NON_POINTER) ++ continue; ++ ++ isneg = aux->alu_state & BPF_ALU_NEG_VALUE; ++ issrc = (aux->alu_state & BPF_ALU_SANITIZE) == ++ BPF_ALU_SANITIZE_SRC; ++ isimm = aux->alu_state & BPF_ALU_IMMEDIATE; ++ ++ off_reg = issrc ? insn->src_reg : insn->dst_reg; ++ if (isimm) { ++ *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); ++ } else { ++ if (isneg) ++ *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); ++ *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); ++ *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); ++ *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); ++ *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); ++ *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); ++ *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); ++ } ++ if (!issrc) ++ *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); ++ insn->src_reg = BPF_REG_AX; ++ if (isneg) ++ insn->code = insn->code == code_add ? ++ code_sub : code_add; ++ *patch++ = *insn; ++ if (issrc && isneg && !isimm) ++ *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); ++ cnt = patch - insn_buf; ++ ++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ if (insn->code != (BPF_JMP | BPF_CALL)) ++ continue; ++ if (insn->src_reg == BPF_PSEUDO_CALL) ++ continue; ++ ++ if (insn->imm == BPF_FUNC_get_route_realm) ++ prog->dst_needed = 1; ++ if (insn->imm == BPF_FUNC_get_prandom_u32) ++ bpf_user_rnd_init_once(); ++ if (insn->imm == BPF_FUNC_override_return) ++ prog->kprobe_override = 1; ++ if (insn->imm == BPF_FUNC_tail_call) { ++ /* If we tail call into other programs, we ++ * cannot make any assumptions since they can ++ * be replaced dynamically during runtime in ++ * the program array. ++ */ ++ prog->cb_access = 1; ++ env->prog->aux->stack_depth = MAX_BPF_STACK; ++ env->prog->aux->max_pkt_offset = MAX_PACKET_OFF; ++ ++ /* mark bpf_tail_call as different opcode to avoid ++ * conditional branch in the interpeter for every normal ++ * call and to prevent accidental JITing by JIT compiler ++ * that doesn't support bpf_tail_call yet ++ */ ++ insn->imm = 0; ++ insn->code = BPF_JMP | BPF_TAIL_CALL; ++ ++ aux = &env->insn_aux_data[i + delta]; ++ if (!bpf_map_ptr_unpriv(aux)) ++ continue; ++ ++ /* instead of changing every JIT dealing with tail_call ++ * emit two extra insns: ++ * if (index >= max_entries) goto out; ++ * index &= array->index_mask; ++ * to avoid out-of-bounds cpu speculation ++ */ ++ if (bpf_map_ptr_poisoned(aux)) { ++ verbose(env, "tail_call abusing map_ptr\n"); ++ return -EINVAL; ++ } ++ ++ map_ptr = BPF_MAP_PTR(aux->map_state); ++ insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, ++ map_ptr->max_entries, 2); ++ insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, ++ container_of(map_ptr, ++ struct bpf_array, ++ map)->index_mask); ++ insn_buf[2] = *insn; ++ cnt = 3; ++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup ++ * and other inlining handlers are currently limited to 64 bit ++ * only. ++ */ ++ if (prog->jit_requested && BITS_PER_LONG == 64 && ++ (insn->imm == BPF_FUNC_map_lookup_elem || ++ insn->imm == BPF_FUNC_map_update_elem || ++ insn->imm == BPF_FUNC_map_delete_elem || ++ insn->imm == BPF_FUNC_map_push_elem || ++ insn->imm == BPF_FUNC_map_pop_elem || ++ insn->imm == BPF_FUNC_map_peek_elem)) { ++ aux = &env->insn_aux_data[i + delta]; ++ if (bpf_map_ptr_poisoned(aux)) ++ goto patch_call_imm; ++ ++ map_ptr = BPF_MAP_PTR(aux->map_state); ++ ops = map_ptr->ops; ++ if (insn->imm == BPF_FUNC_map_lookup_elem && ++ ops->map_gen_lookup) { ++ cnt = ops->map_gen_lookup(map_ptr, insn_buf); ++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { ++ verbose(env, "bpf verifier is misconfigured\n"); ++ return -EINVAL; ++ } ++ ++ new_prog = bpf_patch_insn_data(env, i + delta, ++ insn_buf, cnt); ++ if (!new_prog) ++ return -ENOMEM; ++ ++ delta += cnt - 1; ++ env->prog = prog = new_prog; ++ insn = new_prog->insnsi + i + delta; ++ continue; ++ } ++ ++ BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, ++ (void *(*)(struct bpf_map *map, void *key))NULL)); ++ BUILD_BUG_ON(!__same_type(ops->map_delete_elem, ++ (int (*)(struct bpf_map *map, void *key))NULL)); ++ BUILD_BUG_ON(!__same_type(ops->map_update_elem, ++ (int (*)(struct bpf_map *map, void *key, void *value, ++ u64 flags))NULL)); ++ BUILD_BUG_ON(!__same_type(ops->map_push_elem, ++ (int (*)(struct bpf_map *map, void *value, ++ u64 flags))NULL)); ++ BUILD_BUG_ON(!__same_type(ops->map_pop_elem, ++ (int (*)(struct bpf_map *map, void *value))NULL)); ++ BUILD_BUG_ON(!__same_type(ops->map_peek_elem, ++ (int (*)(struct bpf_map *map, void *value))NULL)); ++ ++ switch (insn->imm) { ++ case BPF_FUNC_map_lookup_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - ++ __bpf_call_base; ++ continue; ++ case BPF_FUNC_map_update_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_update_elem) - ++ __bpf_call_base; ++ continue; ++ case BPF_FUNC_map_delete_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - ++ __bpf_call_base; ++ continue; ++ case BPF_FUNC_map_push_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_push_elem) - ++ __bpf_call_base; ++ continue; ++ case BPF_FUNC_map_pop_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - ++ __bpf_call_base; ++ continue; ++ case BPF_FUNC_map_peek_elem: ++ insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - ++ __bpf_call_base; ++ continue; ++ } ++ ++ goto patch_call_imm; ++ } ++ ++patch_call_imm: ++ fn = env->ops->get_func_proto(insn->imm, env->prog); ++ /* all functions that have prototype and verifier allowed ++ * programs to call them, must be real in-kernel functions ++ */ ++ if (!fn->func) { ++ verbose(env, ++ "kernel subsystem misconfigured func %s#%d\n", ++ func_id_name(insn->imm), insn->imm); ++ return -EFAULT; ++ } ++ insn->imm = fn->func - __bpf_call_base; ++ } ++ ++ return 0; ++} ++ ++static void free_states(struct bpf_verifier_env *env) ++{ ++ struct bpf_verifier_state_list *sl, *sln; + int i; + ++ sl = env->free_list; ++ while (sl) { ++ sln = sl->next; ++ free_verifier_state(&sl->state, false); ++ kfree(sl); ++ sl = sln; ++ } ++ + if (!env->explored_states) + return; + +- for (i = 0; i < env->prog->len; i++) { ++ for (i = 0; i < state_htab_size(env); i++) { + sl = env->explored_states[i]; + +- if (sl) +- while (sl != STATE_LIST_MARK) { +- sln = sl->next; +- kfree(sl); +- sl = sln; +- } ++ while (sl) { ++ sln = sl->next; ++ free_verifier_state(&sl->state, false); ++ kfree(sl); ++ sl = sln; ++ } + } + +- kfree(env->explored_states); ++ kvfree(env->explored_states); + } + +-int bpf_check(struct bpf_prog **prog, union bpf_attr *attr) ++static void print_verification_stats(struct bpf_verifier_env *env) + { +- char __user *log_ubuf = NULL; +- struct verifier_env *env; +- int ret = -EINVAL; ++ int i; + +- if ((*prog)->len <= 0 || (*prog)->len > BPF_MAXINSNS) +- return -E2BIG; ++ if (env->log.level & BPF_LOG_STATS) { ++ verbose(env, "verification time %lld usec\n", ++ div_u64(env->verification_time, 1000)); ++ verbose(env, "stack depth "); ++ for (i = 0; i < env->subprog_cnt; i++) { ++ u32 depth = env->subprog_info[i].stack_depth; ++ ++ verbose(env, "%d", depth); ++ if (i + 1 < env->subprog_cnt) ++ verbose(env, "+"); ++ } ++ verbose(env, "\n"); ++ } ++ verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " ++ "total_states %d peak_states %d mark_read %d\n", ++ env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, ++ env->max_states_per_insn, env->total_states, ++ env->peak_states, env->longest_mark_read_walk); ++} ++ ++int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, ++ union bpf_attr __user *uattr) ++{ ++ u64 start_time = ktime_get_ns(); ++ struct bpf_verifier_env *env; ++ struct bpf_verifier_log *log; ++ int i, len, ret = -EINVAL; ++ bool is_priv; + +- /* 'struct verifier_env' can be global, but since it's not small, ++ /* no program is valid */ ++ if (ARRAY_SIZE(bpf_verifier_ops) == 0) ++ return -EINVAL; ++ ++ /* 'struct bpf_verifier_env' can be global, but since it's not small, + * allocate/free it every time bpf_check() is called + */ +- env = kzalloc(sizeof(struct verifier_env), GFP_KERNEL); ++ env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); + if (!env) + return -ENOMEM; ++ log = &env->log; + ++ len = (*prog)->len; ++ env->insn_aux_data = ++ vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); ++ ret = -ENOMEM; ++ if (!env->insn_aux_data) ++ goto err_free_env; ++ for (i = 0; i < len; i++) ++ env->insn_aux_data[i].orig_idx = i; + env->prog = *prog; ++ env->ops = bpf_verifier_ops[env->prog->type]; ++ is_priv = capable(CAP_SYS_ADMIN); + + /* grab the mutex to protect few globals used by verifier */ +- mutex_lock(&bpf_verifier_lock); ++ if (!is_priv) ++ mutex_lock(&bpf_verifier_lock); + + if (attr->log_level || attr->log_buf || attr->log_size) { + /* user requested verbose verifier output + * and supplied buffer to store the verification trace + */ +- log_level = attr->log_level; +- log_ubuf = (char __user *) (unsigned long) attr->log_buf; +- log_size = attr->log_size; +- log_len = 0; ++ log->level = attr->log_level; ++ log->ubuf = (char __user *) (unsigned long) attr->log_buf; ++ log->len_total = attr->log_size; + + ret = -EINVAL; +- /* log_* values have to be sane */ +- if (log_size < 128 || log_size > UINT_MAX >> 8 || +- log_level == 0 || log_ubuf == NULL) +- goto free_env; +- +- ret = -ENOMEM; +- log_buf = vmalloc(log_size); +- if (!log_buf) +- goto free_env; +- } else { +- log_level = 0; +- } ++ /* log attributes have to be sane */ ++ if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || ++ !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) ++ goto err_unlock; ++ } ++ ++ env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); ++ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) ++ env->strict_alignment = true; ++ if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) ++ env->strict_alignment = false; ++ ++ env->allow_ptr_leaks = is_priv; ++ ++ if (is_priv) ++ env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; + + ret = replace_map_fd_with_map_ptr(env); + if (ret < 0) + goto skip_full_check; + +- env->explored_states = kcalloc(env->prog->len, +- sizeof(struct verifier_state_list *), ++ env->explored_states = kcalloc(state_htab_size(env), ++ sizeof(struct bpf_verifier_state_list *), + GFP_USER); + ret = -ENOMEM; + if (!env->explored_states) + goto skip_full_check; + +- ret = check_cfg(env); ++ ret = check_subprogs(env); ++ if (ret < 0) ++ goto skip_full_check; ++ ++ ret = check_btf_info(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + +- env->allow_ptr_leaks = capable(CAP_SYS_ADMIN); ++ ret = check_cfg(env); ++ if (ret < 0) ++ goto skip_full_check; + + ret = do_check(env); ++ if (env->cur_state) { ++ free_verifier_state(env->cur_state, true); ++ env->cur_state = NULL; ++ } + + skip_full_check: +- while (pop_stack(env, NULL) >= 0); ++ while (!pop_stack(env, NULL, NULL)); + free_states(env); + + if (ret == 0) ++ ret = check_max_stack_depth(env); ++ ++ /* instruction rewrites happen after this point */ ++ if (is_priv) { ++ if (ret == 0) ++ opt_hard_wire_dead_code_branches(env); ++ if (ret == 0) ++ ret = opt_remove_dead_code(env); ++ if (ret == 0) ++ ret = opt_remove_nops(env); ++ } else { ++ if (ret == 0) ++ sanitize_dead_code(env); ++ } ++ ++ if (ret == 0) + /* program is valid, convert *(u32*)(ctx + off) accesses */ + ret = convert_ctx_accesses(env); + +- if (log_level && log_len >= log_size - 1) { +- BUG_ON(log_len >= log_size); +- /* verifier log exceeded user supplied buffer */ +- ret = -ENOSPC; +- /* fall through to return what was recorded */ ++ if (ret == 0) ++ ret = fixup_bpf_calls(env); ++ ++ /* do 32-bit optimization after insn patching has done so those patched ++ * insns could be handled correctly. ++ */ ++ if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { ++ ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); ++ env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret ++ : false; + } + +- /* copy verifier log back to user space including trailing zero */ +- if (log_level && copy_to_user(log_ubuf, log_buf, log_len + 1) != 0) { ++ if (ret == 0) ++ ret = fixup_call_args(env); ++ ++ env->verification_time = ktime_get_ns() - start_time; ++ print_verification_stats(env); ++ ++ if (log->level && bpf_verifier_log_full(log)) ++ ret = -ENOSPC; ++ if (log->level && !log->ubuf) { + ret = -EFAULT; +- goto free_log_buf; ++ goto err_release_maps; + } + + if (ret == 0 && env->used_map_cnt) { +@@ -2297,7 +9580,7 @@ skip_full_check: + + if (!env->prog->aux->used_maps) { + ret = -ENOMEM; +- goto free_log_buf; ++ goto err_release_maps; + } + + memcpy(env->prog->aux->used_maps, env->used_maps, +@@ -2310,17 +9593,21 @@ skip_full_check: + convert_pseudo_ld_imm64(env); + } + +-free_log_buf: +- if (log_level) +- vfree(log_buf); +-free_env: ++ if (ret == 0) ++ adjust_btf_func(env); ++ ++err_release_maps: + if (!env->prog->aux->used_maps) + /* if we didn't copy map pointers into bpf_prog_info, release +- * them now. Otherwise free_bpf_prog_info() will release them. ++ * them now. Otherwise free_used_maps() will release them. + */ + release_maps(env); + *prog = env->prog; ++err_unlock: ++ if (!is_priv) ++ mutex_unlock(&bpf_verifier_lock); ++ vfree(env->insn_aux_data); ++err_free_env: + kfree(env); +- mutex_unlock(&bpf_verifier_lock); + return ret; + } +--- a/include/linux/filter.h ++++ b/include/linux/filter.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ + /* + * Linux Socket Filter Data Structures + */ +@@ -7,16 +8,22 @@ + #include + + #include ++#include + #include + #include + #include + #include + #include + #include +-#include ++#include ++#include ++#include ++#include ++#include + +-#include ++#include + ++#include + #include + #include + +@@ -24,6 +31,11 @@ struct sk_buff; + struct sock; + struct seccomp_data; + struct bpf_prog_aux; ++struct xdp_rxq_info; ++struct xdp_buff; ++struct sock_reuseport; ++struct ctl_table; ++struct ctl_table_header; + + /* ArgX, context and stack frame pointer register positions. Note, + * Arg1, Arg2, Arg3, etc are used as argument mappings of function +@@ -40,7 +52,26 @@ struct bpf_prog_aux; + /* Additional register mappings for converted user programs. */ + #define BPF_REG_A BPF_REG_0 + #define BPF_REG_X BPF_REG_7 +-#define BPF_REG_TMP BPF_REG_8 ++#define BPF_REG_TMP BPF_REG_2 /* scratch reg */ ++#define BPF_REG_D BPF_REG_8 /* data, callee-saved */ ++#define BPF_REG_H BPF_REG_9 /* hlen, callee-saved */ ++ ++/* Kernel hidden auxiliary/helper register. */ ++#define BPF_REG_AX MAX_BPF_REG ++#define MAX_BPF_EXT_REG (MAX_BPF_REG + 1) ++#define MAX_BPF_JIT_REG MAX_BPF_EXT_REG ++ ++/* unused opcode to mark special call to bpf_tail_call() helper */ ++#define BPF_TAIL_CALL 0xf0 ++ ++/* unused opcode to mark call to interpreter with arguments */ ++#define BPF_CALL_ARGS 0xe0 ++ ++/* As per nm, we expose JITed images as text (code) section for ++ * kallsyms. That way, tools like perf can find it to match ++ * addresses. ++ */ ++#define BPF_SYM_ELF_TYPE 't' + + /* BPF program can access up to 512 bytes of stack space. */ + #define MAX_BPF_STACK 512 +@@ -129,6 +160,20 @@ struct bpf_prog_aux; + .off = 0, \ + .imm = IMM }) + ++/* Special form of mov32, used for doing explicit zero extension on dst. */ ++#define BPF_ZEXT_REG(DST) \ ++ ((struct bpf_insn) { \ ++ .code = BPF_ALU | BPF_MOV | BPF_X, \ ++ .dst_reg = DST, \ ++ .src_reg = DST, \ ++ .off = 0, \ ++ .imm = 1 }) ++ ++static inline bool insn_is_zext(const struct bpf_insn *insn) ++{ ++ return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; ++} ++ + /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */ + #define BPF_LD_IMM64(DST, IMM) \ + BPF_LD_IMM64_RAW(DST, 0, IMM) +@@ -249,8 +294,51 @@ struct bpf_prog_aux; + .off = OFF, \ + .imm = IMM }) + ++/* Like BPF_JMP_REG, but with 32-bit wide operands for comparison. */ ++ ++#define BPF_JMP32_REG(OP, DST, SRC, OFF) \ ++ ((struct bpf_insn) { \ ++ .code = BPF_JMP32 | BPF_OP(OP) | BPF_X, \ ++ .dst_reg = DST, \ ++ .src_reg = SRC, \ ++ .off = OFF, \ ++ .imm = 0 }) ++ ++/* Like BPF_JMP_IMM, but with 32-bit wide operands for comparison. */ ++ ++#define BPF_JMP32_IMM(OP, DST, IMM, OFF) \ ++ ((struct bpf_insn) { \ ++ .code = BPF_JMP32 | BPF_OP(OP) | BPF_K, \ ++ .dst_reg = DST, \ ++ .src_reg = 0, \ ++ .off = OFF, \ ++ .imm = IMM }) ++ ++/* Unconditional jumps, goto pc + off16 */ ++ ++#define BPF_JMP_A(OFF) \ ++ ((struct bpf_insn) { \ ++ .code = BPF_JMP | BPF_JA, \ ++ .dst_reg = 0, \ ++ .src_reg = 0, \ ++ .off = OFF, \ ++ .imm = 0 }) ++ ++/* Relative call */ ++ ++#define BPF_CALL_REL(TGT) \ ++ ((struct bpf_insn) { \ ++ .code = BPF_JMP | BPF_CALL, \ ++ .dst_reg = 0, \ ++ .src_reg = BPF_PSEUDO_CALL, \ ++ .off = 0, \ ++ .imm = TGT }) ++ + /* Function call */ + ++#define BPF_CAST_CALL(x) \ ++ ((u64 (*)(u64, u64, u64, u64, u64))(x)) ++ + #define BPF_EMIT_CALL(FUNC) \ + ((struct bpf_insn) { \ + .code = BPF_JMP | BPF_CALL, \ +@@ -303,6 +391,112 @@ struct bpf_prog_aux; + bpf_size; \ + }) + ++#define bpf_size_to_bytes(bpf_size) \ ++({ \ ++ int bytes = -EINVAL; \ ++ \ ++ if (bpf_size == BPF_B) \ ++ bytes = sizeof(u8); \ ++ else if (bpf_size == BPF_H) \ ++ bytes = sizeof(u16); \ ++ else if (bpf_size == BPF_W) \ ++ bytes = sizeof(u32); \ ++ else if (bpf_size == BPF_DW) \ ++ bytes = sizeof(u64); \ ++ \ ++ bytes; \ ++}) ++ ++#define BPF_SIZEOF(type) \ ++ ({ \ ++ const int __size = bytes_to_bpf_size(sizeof(type)); \ ++ BUILD_BUG_ON(__size < 0); \ ++ __size; \ ++ }) ++ ++#define BPF_FIELD_SIZEOF(type, field) \ ++ ({ \ ++ const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \ ++ BUILD_BUG_ON(__size < 0); \ ++ __size; \ ++ }) ++ ++#define BPF_LDST_BYTES(insn) \ ++ ({ \ ++ const int __size = bpf_size_to_bytes(BPF_SIZE((insn)->code)); \ ++ WARN_ON(__size < 0); \ ++ __size; \ ++ }) ++ ++#define __BPF_MAP_0(m, v, ...) v ++#define __BPF_MAP_1(m, v, t, a, ...) m(t, a) ++#define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__) ++#define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__) ++#define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__) ++#define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__) ++ ++#define __BPF_REG_0(...) __BPF_PAD(5) ++#define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4) ++#define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3) ++#define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2) ++#define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1) ++#define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__) ++ ++#define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__) ++#define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__) ++ ++#define __BPF_CAST(t, a) \ ++ (__force t) \ ++ (__force \ ++ typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long), \ ++ (unsigned long)0, (t)0))) a ++#define __BPF_V void ++#define __BPF_N ++ ++#define __BPF_DECL_ARGS(t, a) t a ++#define __BPF_DECL_REGS(t, a) u64 a ++ ++#define __BPF_PAD(n) \ ++ __BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ ++ u64, __ur_3, u64, __ur_4, u64, __ur_5) ++ ++#define BPF_CALL_x(x, name, ...) \ ++ static __always_inline \ ++ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ ++ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ ++ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ ++ { \ ++ return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ ++ } \ ++ static __always_inline \ ++ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) ++ ++#define BPF_CALL_0(name, ...) BPF_CALL_x(0, name, __VA_ARGS__) ++#define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__) ++#define BPF_CALL_2(name, ...) BPF_CALL_x(2, name, __VA_ARGS__) ++#define BPF_CALL_3(name, ...) BPF_CALL_x(3, name, __VA_ARGS__) ++#define BPF_CALL_4(name, ...) BPF_CALL_x(4, name, __VA_ARGS__) ++#define BPF_CALL_5(name, ...) BPF_CALL_x(5, name, __VA_ARGS__) ++ ++#define bpf_ctx_range(TYPE, MEMBER) \ ++ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 ++#define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2) \ ++ offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1 ++#if BITS_PER_LONG == 64 ++# define bpf_ctx_range_ptr(TYPE, MEMBER) \ ++ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 ++#else ++# define bpf_ctx_range_ptr(TYPE, MEMBER) \ ++ offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1 ++#endif /* BITS_PER_LONG == 64 */ ++ ++#define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE) \ ++ ({ \ ++ BUILD_BUG_ON(FIELD_SIZEOF(TYPE, MEMBER) != (SIZE)); \ ++ *(PTR_SIZE) = (SIZE); \ ++ offsetof(TYPE, MEMBER); \ ++ }) ++ + #ifdef CONFIG_COMPAT + /* A struct sock_filter is architecture independent. */ + struct compat_sock_fprog { +@@ -317,24 +511,33 @@ struct sock_fprog_kern { + }; + + struct bpf_binary_header { +- unsigned int pages; +- u8 image[]; ++ u32 pages; ++ /* Some arches need word alignment for their instructions */ ++ u8 image[] __aligned(4); + }; + + struct bpf_prog { + u16 pages; /* Number of allocated pages */ +- kmemcheck_bitfield_begin(meta); + u16 jited:1, /* Is our filter JIT'ed? */ ++ jit_requested:1,/* archs need to JIT the prog */ ++ undo_set_mem:1, /* Passed set_memory_ro() checkpoint */ + gpl_compatible:1, /* Is filter GPL compatible? */ + cb_access:1, /* Is control block accessed? */ +- dst_needed:1; /* Do we need dst entry? */ +- kmemcheck_bitfield_end(meta); +- u32 len; /* Number of filter blocks */ ++ dst_needed:1, /* Do we need dst entry? */ ++ blinded:1, /* Was blinded */ ++ is_func:1, /* program is a bpf function */ ++ kprobe_override:1, /* Do we override a kprobe? */ ++ has_callchain_buf:1, /* callchain buffer allocated? */ ++ enforce_expected_attach_type:1; /* Enforce expected_attach_type checking at attach time */ + enum bpf_prog_type type; /* Type of BPF program */ ++ enum bpf_attach_type expected_attach_type; /* For some prog types */ ++ u32 len; /* Number of filter blocks */ ++ u32 jited_len; /* Size of jited insns in bytes */ ++ u8 tag[BPF_TAG_SIZE]; + struct bpf_prog_aux *aux; /* Auxiliary fields */ + struct sock_fprog_kern *orig_prog; /* Original BPF program */ +- unsigned int (*bpf_func)(const struct sk_buff *skb, +- const struct bpf_insn *filter); ++ unsigned int (*bpf_func)(const void *ctx, ++ const struct bpf_insn *insn); + /* Instructions for interpreter */ + union { + struct sock_filter insns[0]; +@@ -343,44 +546,160 @@ struct bpf_prog { + }; + + struct sk_filter { +- atomic_t refcnt; ++ refcount_t refcnt; + struct rcu_head rcu; + struct bpf_prog *prog; + }; + +-#define BPF_PROG_RUN(filter, ctx) (*filter->bpf_func)(ctx, filter->insnsi) ++#define BPF_PROG_RUN(prog, ctx) ({ \ ++ u32 ret; \ ++ ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \ ++ ret; }) ++ ++#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN ++ ++struct bpf_skb_data_end { ++ struct qdisc_skb_cb qdisc_cb; ++ void *data_meta; ++ void *data_end; ++}; + +-static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog, +- struct sk_buff *skb) ++struct bpf_redirect_info { ++ u32 flags; ++ u32 tgt_index; ++ void *tgt_value; ++ struct bpf_map *map; ++ struct bpf_map *map_to_flush; ++ u32 kern_flags; ++}; ++ ++DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); ++ ++/* flags for bpf_redirect_info kern_flags */ ++#define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */ ++ ++/* Compute the linear packet data range [data, data_end) which ++ * will be accessed by various program types (cls_bpf, act_bpf, ++ * lwt, ...). Subsystems allowing direct data access must (!) ++ * ensure that cb[] area can be written to when BPF program is ++ * invoked (otherwise cb[] save/restore is necessary). ++ */ ++static inline void bpf_compute_data_pointers(struct sk_buff *skb) + { +- u8 *cb_data = qdisc_skb_cb(skb)->data; +- u8 saved_cb[QDISC_CB_PRIV_LEN]; +- u32 res; ++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; ++ ++ BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb)); ++ cb->data_meta = skb->data; ++ cb->data_end = skb->data + skb_headlen(skb); ++} + ++/* Similar to bpf_compute_data_pointers(), except that save orginal ++ * data in cb->data and cb->meta_data for restore. ++ */ ++static inline void bpf_compute_and_save_data_end( ++ struct sk_buff *skb, void **saved_data_end) ++{ ++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; ++ ++ *saved_data_end = cb->data_end; ++ cb->data_end = skb->data + skb_headlen(skb); ++} ++ ++/* Restore data saved by bpf_compute_data_pointers(). */ ++static inline void bpf_restore_data_end( ++ struct sk_buff *skb, void *saved_data_end) ++{ ++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; ++ ++ cb->data_end = saved_data_end; ++} ++ ++static inline u8 *bpf_skb_cb(struct sk_buff *skb) ++{ ++ /* eBPF programs may read/write skb->cb[] area to transfer meta ++ * data between tail calls. Since this also needs to work with ++ * tc, that scratch memory is mapped to qdisc_skb_cb's data area. ++ * ++ * In some socket filter cases, the cb unfortunately needs to be ++ * saved/restored so that protocol specific skb->cb[] data won't ++ * be lost. In any case, due to unpriviledged eBPF programs ++ * attached to sockets, we need to clear the bpf_skb_cb() area ++ * to not leak previous contents to user space. ++ */ ++ BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN); + BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != +- QDISC_CB_PRIV_LEN); ++ FIELD_SIZEOF(struct qdisc_skb_cb, data)); ++ ++ return qdisc_skb_cb(skb)->data; ++} ++ ++static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog, ++ struct sk_buff *skb) ++{ ++ u8 *cb_data = bpf_skb_cb(skb); ++ u8 cb_saved[BPF_SKB_CB_LEN]; ++ u32 res; + + if (unlikely(prog->cb_access)) { +- memcpy(saved_cb, cb_data, sizeof(saved_cb)); +- memset(cb_data, 0, sizeof(saved_cb)); ++ memcpy(cb_saved, cb_data, sizeof(cb_saved)); ++ memset(cb_data, 0, sizeof(cb_saved)); + } + + res = BPF_PROG_RUN(prog, skb); + + if (unlikely(prog->cb_access)) +- memcpy(cb_data, saved_cb, sizeof(saved_cb)); ++ memcpy(cb_data, cb_saved, sizeof(cb_saved)); ++ ++ return res; ++} ++ ++static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog, ++ struct sk_buff *skb) ++{ ++ u32 res; + ++ preempt_disable(); ++ res = __bpf_prog_run_save_cb(prog, skb); ++ preempt_enable(); + return res; + } + + static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog, + struct sk_buff *skb) + { +- u8 *cb_data = qdisc_skb_cb(skb)->data; ++ u8 *cb_data = bpf_skb_cb(skb); ++ u32 res; + + if (unlikely(prog->cb_access)) +- memset(cb_data, 0, QDISC_CB_PRIV_LEN); +- return BPF_PROG_RUN(prog, skb); ++ memset(cb_data, 0, BPF_SKB_CB_LEN); ++ ++ preempt_disable(); ++ res = BPF_PROG_RUN(prog, skb); ++ preempt_enable(); ++ return res; ++} ++ ++static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog, ++ struct xdp_buff *xdp) ++{ ++ /* Caller needs to hold rcu_read_lock() (!), otherwise program ++ * can be released while still running, or map elements could be ++ * freed early while still having concurrent users. XDP fastpath ++ * already takes rcu_read_lock() when fetching the program, so ++ * it's not necessary here anymore. ++ */ ++ return BPF_PROG_RUN(prog, xdp); ++} ++ ++static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog) ++{ ++ return prog->len * sizeof(struct bpf_insn); ++} ++ ++static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog) ++{ ++ return round_up(bpf_prog_insn_size(prog) + ++ sizeof(__be64) + 1, SHA_MESSAGE_BYTES); + } + + static inline unsigned int bpf_prog_size(unsigned int proglen) +@@ -399,27 +718,77 @@ static inline bool bpf_prog_was_classic( + return prog->type == BPF_PROG_TYPE_UNSPEC; + } + ++static inline u32 bpf_ctx_off_adjust_machine(u32 size) ++{ ++ const u32 size_machine = sizeof(unsigned long); ++ ++ if (size > size_machine && size % size_machine == 0) ++ size = size_machine; ++ ++ return size; ++} ++ ++static inline bool ++bpf_ctx_narrow_access_ok(u32 off, u32 size, u32 size_default) ++{ ++ return size <= size_default && (size & (size - 1)) == 0; ++} ++ ++static inline u8 ++bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default) ++{ ++ u8 access_off = off & (size_default - 1); ++ ++#ifdef __LITTLE_ENDIAN ++ return access_off; ++#else ++ return size_default - (access_off + size); ++#endif ++} ++ ++#define bpf_ctx_wide_access_ok(off, size, type, field) \ ++ (size == sizeof(__u64) && \ ++ off >= offsetof(type, field) && \ ++ off + sizeof(__u64) <= offsetofend(type, field) && \ ++ off % sizeof(__u64) == 0) ++ + #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0])) + +-#ifdef CONFIG_DEBUG_SET_MODULE_RONX + static inline void bpf_prog_lock_ro(struct bpf_prog *fp) + { +- set_memory_ro((unsigned long)fp, fp->pages); ++#ifndef CONFIG_BPF_JIT_ALWAYS_ON ++ if (!fp->jited) { ++ fp->undo_set_mem = 1; ++ set_memory_ro((unsigned long)fp, fp->pages); ++ } ++#endif + } + + static inline void bpf_prog_unlock_ro(struct bpf_prog *fp) + { +- set_memory_rw((unsigned long)fp, fp->pages); ++ if (fp->undo_set_mem) ++ set_memory_rw((unsigned long)fp, fp->pages); + } +-#else +-static inline void bpf_prog_lock_ro(struct bpf_prog *fp) ++ ++static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr) + { ++ set_memory_ro((unsigned long)hdr, hdr->pages); ++ set_memory_x((unsigned long)hdr, hdr->pages); + } + +-static inline void bpf_prog_unlock_ro(struct bpf_prog *fp) ++static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr) ++{ ++ set_memory_rw((unsigned long)hdr, hdr->pages); ++} ++ ++static inline struct bpf_binary_header * ++bpf_jit_binary_hdr(const struct bpf_prog *fp) + { ++ unsigned long real_start = (unsigned long)fp->bpf_func; ++ unsigned long addr = real_start & PAGE_MASK; ++ ++ return (void *)addr; + } +-#endif /* CONFIG_DEBUG_SET_MODULE_RONX */ + + int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap); + static inline int sk_filter(struct sock *sk, struct sk_buff *skb) +@@ -427,10 +796,20 @@ static inline int sk_filter(struct sock + return sk_filter_trim_cap(sk, skb, 1); + } + +-int bpf_prog_select_runtime(struct bpf_prog *fp); ++struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); + void bpf_prog_free(struct bpf_prog *fp); + ++bool bpf_opcode_in_insntable(u8 code); ++ ++void bpf_prog_free_linfo(struct bpf_prog *prog); ++void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, ++ const u32 *insn_to_jit_off); ++int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog); ++void bpf_prog_free_jited_linfo(struct bpf_prog *prog); ++void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog); ++ + struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags); ++struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags); + struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, + gfp_t gfp_extra_flags); + void __bpf_prog_free(struct bpf_prog *fp); +@@ -450,12 +829,11 @@ int bpf_prog_create_from_user(struct bpf + void bpf_prog_destroy(struct bpf_prog *fp); + + int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk); +-int __sk_attach_filter(struct sock_fprog *fprog, struct sock *sk, +- bool locked); + int sk_attach_bpf(u32 ufd, struct sock *sk); ++int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk); ++int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk); ++void sk_reuseport_prog_free(struct bpf_prog *prog); + int sk_detach_filter(struct sock *sk); +-int __sk_detach_filter(struct sock *sk, bool locked); +- + int sk_get_filter(struct sock *sk, struct sock_filter __user *filter, + unsigned int len); + +@@ -463,10 +841,100 @@ bool sk_filter_charge(struct sock *sk, s + void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); + + u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); +-void bpf_int_jit_compile(struct bpf_prog *fp); +-bool bpf_helper_changes_skb_data(void *func); ++#define __bpf_call_base_args \ ++ ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ ++ (void *)__bpf_call_base) ++ ++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); ++void bpf_jit_compile(struct bpf_prog *prog); ++bool bpf_jit_needs_zext(void); ++bool bpf_helper_changes_pkt_data(void *func); ++ ++static inline bool bpf_dump_raw_ok(const struct cred *cred) ++{ ++ /* Reconstruction of call-sites is dependent on kallsyms, ++ * thus make dump the same restriction. ++ */ ++ return true; ++} ++ ++struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, ++ const struct bpf_insn *patch, u32 len); ++int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); ++ ++void bpf_clear_redirect_map(struct bpf_map *map); ++ ++static inline bool xdp_return_frame_no_direct(void) ++{ ++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); ++ ++ return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT; ++} ++ ++static inline void xdp_set_return_frame_no_direct(void) ++{ ++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); ++ ++ ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT; ++} ++ ++static inline void xdp_clear_return_frame_no_direct(void) ++{ ++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); ++ ++ ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT; ++} ++ ++static inline int xdp_ok_fwd_dev(const struct net_device *fwd, ++ unsigned int pktlen) ++{ ++ unsigned int len; ++ ++ if (unlikely(!(fwd->flags & IFF_UP))) ++ return -ENETDOWN; ++ ++ len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN; ++ if (pktlen > len) ++ return -EMSGSIZE; ++ ++ return 0; ++} ++ ++/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the ++ * same cpu context. Further for best results no more than a single map ++ * for the do_redirect/do_flush pair should be used. This limitation is ++ * because we only track one map and force a flush when the map changes. ++ * This does not appear to be a real limitation for existing software. ++ */ ++int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, ++ struct xdp_buff *xdp, struct bpf_prog *prog); ++int xdp_do_redirect(struct net_device *dev, ++ struct xdp_buff *xdp, ++ struct bpf_prog *prog); ++void xdp_do_flush_map(void); ++ ++void bpf_warn_invalid_xdp_action(u32 act); ++ ++#ifdef CONFIG_INET ++struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, ++ struct bpf_prog *prog, struct sk_buff *skb, ++ u32 hash); ++#else ++static inline struct sock * ++bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, ++ struct bpf_prog *prog, struct sk_buff *skb, ++ u32 hash) ++{ ++ return NULL; ++} ++#endif + + #ifdef CONFIG_BPF_JIT ++extern int bpf_jit_enable; ++extern int bpf_jit_harden; ++extern int bpf_jit_kallsyms; ++extern long bpf_jit_limit; ++ + typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); + + struct bpf_binary_header * +@@ -474,10 +942,18 @@ bpf_jit_binary_alloc(unsigned int progle + unsigned int alignment, + bpf_jit_fill_hole_t bpf_fill_ill_insns); + void bpf_jit_binary_free(struct bpf_binary_header *hdr); +- +-void bpf_jit_compile(struct bpf_prog *fp); ++u64 bpf_jit_alloc_exec_limit(void); ++void *bpf_jit_alloc_exec(unsigned long size); ++void bpf_jit_free_exec(void *addr); + void bpf_jit_free(struct bpf_prog *fp); + ++int bpf_jit_get_func_addr(const struct bpf_prog *prog, ++ const struct bpf_insn *insn, bool extra_pass, ++ u64 *func_addr, bool *func_addr_fixed); ++ ++struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp); ++void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); ++ + static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, + u32 pass, void *image) + { +@@ -488,17 +964,144 @@ static inline void bpf_jit_dump(unsigned + print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET, + 16, 1, image, proglen, false); + } +-#else +-static inline void bpf_jit_compile(struct bpf_prog *fp) ++ ++static inline bool bpf_jit_is_ebpf(void) ++{ ++# ifdef CONFIG_HAVE_EBPF_JIT ++ return true; ++# else ++ return false; ++# endif ++} ++ ++static inline bool ebpf_jit_enabled(void) ++{ ++ return bpf_jit_enable && bpf_jit_is_ebpf(); ++} ++ ++static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) ++{ ++ return fp->jited && bpf_jit_is_ebpf(); ++} ++ ++static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) ++{ ++ /* These are the prerequisites, should someone ever have the ++ * idea to call blinding outside of them, we make sure to ++ * bail out. ++ */ ++ if (!bpf_jit_is_ebpf()) ++ return false; ++ if (!prog->jit_requested) ++ return false; ++ if (!bpf_jit_harden) ++ return false; ++ if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN)) ++ return false; ++ ++ return true; ++} ++ ++static inline bool bpf_jit_kallsyms_enabled(void) + { ++ /* There are a couple of corner cases where kallsyms should ++ * not be enabled f.e. on hardening. ++ */ ++ if (bpf_jit_harden) ++ return false; ++ if (!bpf_jit_kallsyms) ++ return false; ++ if (bpf_jit_kallsyms == 1) ++ return true; ++ ++ return false; ++} ++ ++const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, ++ unsigned long *off, char *sym); ++bool is_bpf_text_address(unsigned long addr); ++int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, ++ char *sym); ++ ++static inline const char * ++bpf_address_lookup(unsigned long addr, unsigned long *size, ++ unsigned long *off, char **modname, char *sym) ++{ ++ const char *ret = __bpf_address_lookup(addr, size, off, sym); ++ ++ if (ret && modname) ++ *modname = NULL; ++ return ret; ++} ++ ++void bpf_prog_kallsyms_add(struct bpf_prog *fp); ++void bpf_prog_kallsyms_del(struct bpf_prog *fp); ++void bpf_get_prog_name(const struct bpf_prog *prog, char *sym); ++ ++#else /* CONFIG_BPF_JIT */ ++ ++static inline bool ebpf_jit_enabled(void) ++{ ++ return false; ++} ++ ++static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) ++{ ++ return false; + } + + static inline void bpf_jit_free(struct bpf_prog *fp) + { + bpf_prog_unlock_free(fp); + } ++ ++static inline bool bpf_jit_kallsyms_enabled(void) ++{ ++ return false; ++} ++ ++static inline const char * ++__bpf_address_lookup(unsigned long addr, unsigned long *size, ++ unsigned long *off, char *sym) ++{ ++ return NULL; ++} ++ ++static inline bool is_bpf_text_address(unsigned long addr) ++{ ++ return false; ++} ++ ++static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value, ++ char *type, char *sym) ++{ ++ return -ERANGE; ++} ++ ++static inline const char * ++bpf_address_lookup(unsigned long addr, unsigned long *size, ++ unsigned long *off, char **modname, char *sym) ++{ ++ return NULL; ++} ++ ++static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) ++{ ++} ++ ++static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) ++{ ++} ++ ++static inline void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) ++{ ++ sym[0] = '\0'; ++} ++ + #endif /* CONFIG_BPF_JIT */ + ++void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); ++ + #define BPF_ANC BIT(15) + + static inline bool bpf_needs_clear_a(const struct sock_filter *first) +@@ -571,4 +1174,59 @@ static inline int bpf_tell_extensions(vo + return SKF_AD_MAX; + } + ++struct bpf_sock_addr_kern { ++ struct sock *sk; ++ struct sockaddr *uaddr; ++ /* Temporary "register" to make indirect stores to nested structures ++ * defined above. We need three registers to make such a store, but ++ * only two (src and dst) are available at convert_ctx_access time ++ */ ++ u64 tmp_reg; ++ void *t_ctx; /* Attach type specific context. */ ++}; ++ ++struct bpf_sock_ops_kern { ++ struct sock *sk; ++ u32 op; ++ union { ++ u32 args[4]; ++ u32 reply; ++ u32 replylong[4]; ++ }; ++ u32 is_fullsock; ++ u64 temp; /* temp and everything after is not ++ * initialized to 0 before calling ++ * the BPF program. New fields that ++ * should be initialized to 0 should ++ * be inserted before temp. ++ * temp is scratch storage used by ++ * sock_ops_convert_ctx_access ++ * as temporary storage of a register. ++ */ ++}; ++ ++struct bpf_sysctl_kern { ++ struct ctl_table_header *head; ++ struct ctl_table *table; ++ void *cur_val; ++ size_t cur_len; ++ void *new_val; ++ size_t new_len; ++ int new_updated; ++ int write; ++ loff_t *ppos; ++ /* Temporary "register" for indirect stores to ppos. */ ++ u64 tmp_reg; ++}; ++ ++struct bpf_sockopt_kern { ++ struct sock *sk; ++ u8 *optval; ++ u8 *optval_end; ++ s32 level; ++ s32 optname; ++ s32 optlen; ++ s32 retval; ++}; ++ + #endif /* __LINUX_FILTER_H__ */ +--- /dev/null ++++ b/include/linux/set_memory.h +@@ -0,0 +1,47 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Copyright 2017, Michael Ellerman, IBM Corporation. ++ */ ++#ifndef _LINUX_SET_MEMORY_H_ ++#define _LINUX_SET_MEMORY_H_ ++ ++#include ++ ++#ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP ++static inline int set_direct_map_invalid_noflush(struct page *page) ++{ ++ return 0; ++} ++static inline int set_direct_map_default_noflush(struct page *page) ++{ ++ return 0; ++} ++#endif ++ ++#ifndef set_mce_nospec ++static inline int set_mce_nospec(unsigned long pfn, bool unmap) ++{ ++ return 0; ++} ++#endif ++ ++#ifndef clear_mce_nospec ++static inline int clear_mce_nospec(unsigned long pfn) ++{ ++ return 0; ++} ++#endif ++ ++#ifndef CONFIG_ARCH_HAS_MEM_ENCRYPT ++static inline int set_memory_encrypted(unsigned long addr, int numpages) ++{ ++ return 0; ++} ++ ++static inline int set_memory_decrypted(unsigned long addr, int numpages) ++{ ++ return 0; ++} ++#endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */ ++ ++#endif /* _LINUX_SET_MEMORY_H_ */ +--- /dev/null ++++ b/include/trace/events/xdp.h +@@ -0,0 +1,407 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM xdp ++ ++#if !defined(_TRACE_XDP_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_XDP_H ++ ++#include ++#include ++#include ++#include ++ ++#define __XDP_ACT_MAP(FN) \ ++ FN(ABORTED) \ ++ FN(DROP) \ ++ FN(PASS) \ ++ FN(TX) \ ++ FN(REDIRECT) ++ ++#define __XDP_ACT_TP_FN(x) \ ++ TRACE_DEFINE_ENUM(XDP_##x); ++#define __XDP_ACT_SYM_FN(x) \ ++ { XDP_##x, #x }, ++#define __XDP_ACT_SYM_TAB \ ++ __XDP_ACT_MAP(__XDP_ACT_SYM_FN) { -1, 0 } ++__XDP_ACT_MAP(__XDP_ACT_TP_FN) ++ ++TRACE_EVENT(xdp_exception, ++ ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, u32 act), ++ ++ TP_ARGS(dev, xdp, act), ++ ++ TP_STRUCT__entry( ++ __field(int, prog_id) ++ __field(u32, act) ++ __field(int, ifindex) ++ ), ++ ++ TP_fast_assign( ++ __entry->prog_id = xdp->aux->id; ++ __entry->act = act; ++ __entry->ifindex = dev->ifindex; ++ ), ++ ++ TP_printk("prog_id=%d action=%s ifindex=%d", ++ __entry->prog_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->ifindex) ++); ++ ++TRACE_EVENT(xdp_bulk_tx, ++ ++ TP_PROTO(const struct net_device *dev, ++ int sent, int drops, int err), ++ ++ TP_ARGS(dev, sent, drops, err), ++ ++ TP_STRUCT__entry( ++ __field(int, ifindex) ++ __field(u32, act) ++ __field(int, drops) ++ __field(int, sent) ++ __field(int, err) ++ ), ++ ++ TP_fast_assign( ++ __entry->ifindex = dev->ifindex; ++ __entry->act = XDP_TX; ++ __entry->drops = drops; ++ __entry->sent = sent; ++ __entry->err = err; ++ ), ++ ++ TP_printk("ifindex=%d action=%s sent=%d drops=%d err=%d", ++ __entry->ifindex, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->sent, __entry->drops, __entry->err) ++); ++ ++DECLARE_EVENT_CLASS(xdp_redirect_template, ++ ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, ++ int to_ifindex, int err, ++ const struct bpf_map *map, u32 map_index), ++ ++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), ++ ++ TP_STRUCT__entry( ++ __field(int, prog_id) ++ __field(u32, act) ++ __field(int, ifindex) ++ __field(int, err) ++ __field(int, to_ifindex) ++ __field(u32, map_id) ++ __field(int, map_index) ++ ), ++ ++ TP_fast_assign( ++ __entry->prog_id = xdp->aux->id; ++ __entry->act = XDP_REDIRECT; ++ __entry->ifindex = dev->ifindex; ++ __entry->err = err; ++ __entry->to_ifindex = to_ifindex; ++ __entry->map_id = map ? map->id : 0; ++ __entry->map_index = map_index; ++ ), ++ ++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d", ++ __entry->prog_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->ifindex, __entry->to_ifindex, ++ __entry->err) ++); ++ ++DEFINE_EVENT(xdp_redirect_template, xdp_redirect, ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, ++ int to_ifindex, int err, ++ const struct bpf_map *map, u32 map_index), ++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index) ++); ++ ++DEFINE_EVENT(xdp_redirect_template, xdp_redirect_err, ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, ++ int to_ifindex, int err, ++ const struct bpf_map *map, u32 map_index), ++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index) ++); ++ ++#define _trace_xdp_redirect(dev, xdp, to) \ ++ trace_xdp_redirect(dev, xdp, to, 0, NULL, 0); ++ ++#define _trace_xdp_redirect_err(dev, xdp, to, err) \ ++ trace_xdp_redirect_err(dev, xdp, to, err, NULL, 0); ++ ++DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map, ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, ++ int to_ifindex, int err, ++ const struct bpf_map *map, u32 map_index), ++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), ++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d" ++ " map_id=%d map_index=%d", ++ __entry->prog_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->ifindex, __entry->to_ifindex, ++ __entry->err, ++ __entry->map_id, __entry->map_index) ++); ++ ++DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map_err, ++ TP_PROTO(const struct net_device *dev, ++ const struct bpf_prog *xdp, ++ int to_ifindex, int err, ++ const struct bpf_map *map, u32 map_index), ++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), ++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d" ++ " map_id=%d map_index=%d", ++ __entry->prog_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->ifindex, __entry->to_ifindex, ++ __entry->err, ++ __entry->map_id, __entry->map_index) ++); ++ ++#ifndef __DEVMAP_OBJ_TYPE ++#define __DEVMAP_OBJ_TYPE ++struct _bpf_dtab_netdev { ++ struct net_device *dev; ++}; ++#endif /* __DEVMAP_OBJ_TYPE */ ++ ++#define devmap_ifindex(fwd, map) \ ++ ((map->map_type == BPF_MAP_TYPE_DEVMAP || \ ++ map->map_type == BPF_MAP_TYPE_DEVMAP_HASH) ? \ ++ ((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0) ++ ++#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \ ++ trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \ ++ 0, map, idx) ++ ++#define _trace_xdp_redirect_map_err(dev, xdp, fwd, map, idx, err) \ ++ trace_xdp_redirect_map_err(dev, xdp, devmap_ifindex(fwd, map), \ ++ err, map, idx) ++ ++TRACE_EVENT(xdp_cpumap_kthread, ++ ++ TP_PROTO(int map_id, unsigned int processed, unsigned int drops, ++ int sched), ++ ++ TP_ARGS(map_id, processed, drops, sched), ++ ++ TP_STRUCT__entry( ++ __field(int, map_id) ++ __field(u32, act) ++ __field(int, cpu) ++ __field(unsigned int, drops) ++ __field(unsigned int, processed) ++ __field(int, sched) ++ ), ++ ++ TP_fast_assign( ++ __entry->map_id = map_id; ++ __entry->act = XDP_REDIRECT; ++ __entry->cpu = smp_processor_id(); ++ __entry->drops = drops; ++ __entry->processed = processed; ++ __entry->sched = sched; ++ ), ++ ++ TP_printk("kthread" ++ " cpu=%d map_id=%d action=%s" ++ " processed=%u drops=%u" ++ " sched=%d", ++ __entry->cpu, __entry->map_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->processed, __entry->drops, ++ __entry->sched) ++); ++ ++TRACE_EVENT(xdp_cpumap_enqueue, ++ ++ TP_PROTO(int map_id, unsigned int processed, unsigned int drops, ++ int to_cpu), ++ ++ TP_ARGS(map_id, processed, drops, to_cpu), ++ ++ TP_STRUCT__entry( ++ __field(int, map_id) ++ __field(u32, act) ++ __field(int, cpu) ++ __field(unsigned int, drops) ++ __field(unsigned int, processed) ++ __field(int, to_cpu) ++ ), ++ ++ TP_fast_assign( ++ __entry->map_id = map_id; ++ __entry->act = XDP_REDIRECT; ++ __entry->cpu = smp_processor_id(); ++ __entry->drops = drops; ++ __entry->processed = processed; ++ __entry->to_cpu = to_cpu; ++ ), ++ ++ TP_printk("enqueue" ++ " cpu=%d map_id=%d action=%s" ++ " processed=%u drops=%u" ++ " to_cpu=%d", ++ __entry->cpu, __entry->map_id, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->processed, __entry->drops, ++ __entry->to_cpu) ++); ++ ++TRACE_EVENT(xdp_devmap_xmit, ++ ++ TP_PROTO(const struct bpf_map *map, u32 map_index, ++ int sent, int drops, ++ const struct net_device *from_dev, ++ const struct net_device *to_dev, int err), ++ ++ TP_ARGS(map, map_index, sent, drops, from_dev, to_dev, err), ++ ++ TP_STRUCT__entry( ++ __field(int, map_id) ++ __field(u32, act) ++ __field(u32, map_index) ++ __field(int, drops) ++ __field(int, sent) ++ __field(int, from_ifindex) ++ __field(int, to_ifindex) ++ __field(int, err) ++ ), ++ ++ TP_fast_assign( ++ __entry->map_id = map->id; ++ __entry->act = XDP_REDIRECT; ++ __entry->map_index = map_index; ++ __entry->drops = drops; ++ __entry->sent = sent; ++ __entry->from_ifindex = from_dev->ifindex; ++ __entry->to_ifindex = to_dev->ifindex; ++ __entry->err = err; ++ ), ++ ++ TP_printk("ndo_xdp_xmit" ++ " map_id=%d map_index=%d action=%s" ++ " sent=%d drops=%d" ++ " from_ifindex=%d to_ifindex=%d err=%d", ++ __entry->map_id, __entry->map_index, ++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), ++ __entry->sent, __entry->drops, ++ __entry->from_ifindex, __entry->to_ifindex, __entry->err) ++); ++ ++/* Expect users already include , but not xdp_priv.h */ ++#include ++ ++#define __MEM_TYPE_MAP(FN) \ ++ FN(PAGE_SHARED) \ ++ FN(PAGE_ORDER0) \ ++ FN(PAGE_POOL) \ ++ FN(ZERO_COPY) ++ ++#define __MEM_TYPE_TP_FN(x) \ ++ TRACE_DEFINE_ENUM(MEM_TYPE_##x); ++#define __MEM_TYPE_SYM_FN(x) \ ++ { MEM_TYPE_##x, #x }, ++#define __MEM_TYPE_SYM_TAB \ ++ __MEM_TYPE_MAP(__MEM_TYPE_SYM_FN) { -1, 0 } ++__MEM_TYPE_MAP(__MEM_TYPE_TP_FN) ++ ++TRACE_EVENT(mem_disconnect, ++ ++ TP_PROTO(const struct xdp_mem_allocator *xa), ++ ++ TP_ARGS(xa), ++ ++ TP_STRUCT__entry( ++ __field(const struct xdp_mem_allocator *, xa) ++ __field(u32, mem_id) ++ __field(u32, mem_type) ++ __field(const void *, allocator) ++ ), ++ ++ TP_fast_assign( ++ __entry->xa = xa; ++ __entry->mem_id = xa->mem.id; ++ __entry->mem_type = xa->mem.type; ++ __entry->allocator = xa->allocator; ++ ), ++ ++ TP_printk("mem_id=%d mem_type=%s allocator=%p", ++ __entry->mem_id, ++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), ++ __entry->allocator ++ ) ++); ++ ++TRACE_EVENT(mem_connect, ++ ++ TP_PROTO(const struct xdp_mem_allocator *xa, ++ const struct xdp_rxq_info *rxq), ++ ++ TP_ARGS(xa, rxq), ++ ++ TP_STRUCT__entry( ++ __field(const struct xdp_mem_allocator *, xa) ++ __field(u32, mem_id) ++ __field(u32, mem_type) ++ __field(const void *, allocator) ++ __field(const struct xdp_rxq_info *, rxq) ++ __field(int, ifindex) ++ ), ++ ++ TP_fast_assign( ++ __entry->xa = xa; ++ __entry->mem_id = xa->mem.id; ++ __entry->mem_type = xa->mem.type; ++ __entry->allocator = xa->allocator; ++ __entry->rxq = rxq; ++ __entry->ifindex = rxq->dev->ifindex; ++ ), ++ ++ TP_printk("mem_id=%d mem_type=%s allocator=%p" ++ " ifindex=%d", ++ __entry->mem_id, ++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), ++ __entry->allocator, ++ __entry->ifindex ++ ) ++); ++ ++TRACE_EVENT(mem_return_failed, ++ ++ TP_PROTO(const struct xdp_mem_info *mem, ++ const struct page *page), ++ ++ TP_ARGS(mem, page), ++ ++ TP_STRUCT__entry( ++ __field(const struct page *, page) ++ __field(u32, mem_id) ++ __field(u32, mem_type) ++ ), ++ ++ TP_fast_assign( ++ __entry->page = page; ++ __entry->mem_id = mem->id; ++ __entry->mem_type = mem->type; ++ ), ++ ++ TP_printk("mem_id=%d mem_type=%s page=%p", ++ __entry->mem_id, ++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), ++ __entry->page ++ ) ++); ++ ++#endif /* _TRACE_XDP_H */ ++ ++#include +--- /dev/null ++++ b/include/net/xdp_priv.h +@@ -0,0 +1,20 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++#ifndef __LINUX_NET_XDP_PRIV_H__ ++#define __LINUX_NET_XDP_PRIV_H__ ++ ++#include ++#include ++ ++/* Private to net/core/xdp.c, but used by trace/events/xdp.h */ ++struct xdp_mem_allocator { ++ struct xdp_mem_info mem; ++ union { ++ void *allocator; ++ struct page_pool *page_pool; ++ struct zero_copy_allocator *zc_alloc; ++ }; ++ struct rhash_head node; ++ struct rcu_head rcu; ++}; ++ ++#endif /* __LINUX_NET_XDP_PRIV_H__ */ +--- /dev/null ++++ b/include/net/xdp.h +@@ -0,0 +1,184 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* include/net/xdp.h ++ * ++ * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. ++ */ ++#ifndef __LINUX_NET_XDP_H__ ++#define __LINUX_NET_XDP_H__ ++ ++/** ++ * DOC: XDP RX-queue information ++ * ++ * The XDP RX-queue info (xdp_rxq_info) is associated with the driver ++ * level RX-ring queues. It is information that is specific to how ++ * the driver have configured a given RX-ring queue. ++ * ++ * Each xdp_buff frame received in the driver carry a (pointer) ++ * reference to this xdp_rxq_info structure. This provides the XDP ++ * data-path read-access to RX-info for both kernel and bpf-side ++ * (limited subset). ++ * ++ * For now, direct access is only safe while running in NAPI/softirq ++ * context. Contents is read-mostly and must not be updated during ++ * driver NAPI/softirq poll. ++ * ++ * The driver usage API is a register and unregister API. ++ * ++ * The struct is not directly tied to the XDP prog. A new XDP prog ++ * can be attached as long as it doesn't change the underlying ++ * RX-ring. If the RX-ring does change significantly, the NIC driver ++ * naturally need to stop the RX-ring before purging and reallocating ++ * memory. In that process the driver MUST call unregistor (which ++ * also apply for driver shutdown and unload). The register API is ++ * also mandatory during RX-ring setup. ++ */ ++ ++enum xdp_mem_type { ++ MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ ++ MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ ++ MEM_TYPE_PAGE_POOL, ++ MEM_TYPE_ZERO_COPY, ++ MEM_TYPE_MAX, ++}; ++ ++/* XDP flags for ndo_xdp_xmit */ ++#define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ ++#define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH ++ ++struct xdp_mem_info { ++ u32 type; /* enum xdp_mem_type, but known size type */ ++ u32 id; ++}; ++ ++struct page_pool; ++ ++struct zero_copy_allocator { ++ void (*free)(struct zero_copy_allocator *zca, unsigned long handle); ++}; ++ ++struct xdp_rxq_info { ++ struct net_device *dev; ++ u32 queue_index; ++ u32 reg_state; ++ struct xdp_mem_info mem; ++} ____cacheline_aligned; /* perf critical, avoid false-sharing */ ++ ++struct xdp_buff { ++ void *data; ++ void *data_end; ++ void *data_meta; ++ void *data_hard_start; ++ unsigned long handle; ++ struct xdp_rxq_info *rxq; ++}; ++ ++struct xdp_frame { ++ void *data; ++ u16 len; ++ u16 headroom; ++ u16 metasize; ++ /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, ++ * while mem info is valid on remote CPU. ++ */ ++ struct xdp_mem_info mem; ++ struct net_device *dev_rx; /* used by cpumap */ ++}; ++ ++/* Clear kernel pointers in xdp_frame */ ++static inline void xdp_scrub_frame(struct xdp_frame *frame) ++{ ++ frame->data = NULL; ++ frame->dev_rx = NULL; ++} ++ ++struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); ++ ++/* Convert xdp_buff to xdp_frame */ ++static inline ++struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) ++{ ++ struct xdp_frame *xdp_frame; ++ int metasize; ++ int headroom; ++ ++ if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) ++ return xdp_convert_zc_to_xdp_frame(xdp); ++ ++ /* Assure headroom is available for storing info */ ++ headroom = xdp->data - xdp->data_hard_start; ++ metasize = xdp->data - xdp->data_meta; ++ metasize = metasize > 0 ? metasize : 0; ++ if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) ++ return NULL; ++ ++ /* Store info in top of packet */ ++ xdp_frame = xdp->data_hard_start; ++ ++ xdp_frame->data = xdp->data; ++ xdp_frame->len = xdp->data_end - xdp->data; ++ xdp_frame->headroom = headroom - sizeof(*xdp_frame); ++ xdp_frame->metasize = metasize; ++ ++ /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ ++ xdp_frame->mem = xdp->rxq->mem; ++ ++ return xdp_frame; ++} ++ ++void xdp_return_frame(struct xdp_frame *xdpf); ++void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); ++void xdp_return_buff(struct xdp_buff *xdp); ++ ++/* When sending xdp_frame into the network stack, then there is no ++ * return point callback, which is needed to release e.g. DMA-mapping ++ * resources with page_pool. Thus, have explicit function to release ++ * frame resources. ++ */ ++void __xdp_release_frame(void *data, struct xdp_mem_info *mem); ++static inline void xdp_release_frame(struct xdp_frame *xdpf) ++{ ++ struct xdp_mem_info *mem = &xdpf->mem; ++ ++ /* Curr only page_pool needs this */ ++ if (mem->type == MEM_TYPE_PAGE_POOL) ++ __xdp_release_frame(xdpf->data, mem); ++} ++ ++int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, ++ struct net_device *dev, u32 queue_index); ++void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); ++void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); ++bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); ++int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, ++ enum xdp_mem_type type, void *allocator); ++void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); ++ ++/* Drivers not supporting XDP metadata can use this helper, which ++ * rejects any room expansion for metadata as a result. ++ */ ++static __always_inline void ++xdp_set_data_meta_invalid(struct xdp_buff *xdp) ++{ ++ xdp->data_meta = xdp->data + 1; ++} ++ ++static __always_inline bool ++xdp_data_meta_unsupported(const struct xdp_buff *xdp) ++{ ++ return unlikely(xdp->data_meta > xdp->data); ++} ++ ++struct xdp_attachment_info { ++ struct bpf_prog *prog; ++ u32 flags; ++}; ++ ++struct netdev_bpf; ++int xdp_attachment_query(struct xdp_attachment_info *info, ++ struct netdev_bpf *bpf); ++bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, ++ struct netdev_bpf *bpf); ++void xdp_attachment_setup(struct xdp_attachment_info *info, ++ struct netdev_bpf *bpf); ++ ++#endif /* __LINUX_NET_XDP_H__ */ +--- a/include/linux/atomic.h ++++ b/include/linux/atomic.h +@@ -437,6 +437,8 @@ static inline int atomic_add_unless(atom + return __atomic_add_unless(v, a, u) != u; + } + ++#define atomic_fetch_add_unless __atomic_add_unless ++ + /** + * atomic_inc_not_zero - increment unless the number is zero + * @v: pointer of type atomic_t +--- a/include/linux/kernel.h ++++ b/include/linux/kernel.h +@@ -45,6 +45,13 @@ + + #define STACK_MAGIC 0xdeadbeef + ++#define u64_to_user_ptr(x) ( \ ++ { \ ++ typecheck(u64, (x)); \ ++ (void __user *)(uintptr_t)(x); \ ++ } \ ++) ++ + #define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) + + #define ALIGN(x, a) __ALIGN_KERNEL((x), (a)) +--- /dev/null ++++ b/include/linux/tnum.h +@@ -0,0 +1,89 @@ ++/* tnum: tracked (or tristate) numbers ++ * ++ * A tnum tracks knowledge about the bits of a value. Each bit can be either ++ * known (0 or 1), or unknown (x). Arithmetic operations on tnums will ++ * propagate the unknown bits such that the tnum result represents all the ++ * possible results for possible values of the operands. ++ */ ++ ++#ifndef _LINUX_TNUM_H ++#define _LINUX_TNUM_H ++ ++#include ++ ++struct tnum { ++ u64 value; ++ u64 mask; ++}; ++ ++/* Constructors */ ++/* Represent a known constant as a tnum. */ ++struct tnum tnum_const(u64 value); ++/* A completely unknown value */ ++extern const struct tnum tnum_unknown; ++/* A value that's unknown except that @min <= value <= @max */ ++struct tnum tnum_range(u64 min, u64 max); ++ ++/* Arithmetic and logical ops */ ++/* Shift a tnum left (by a fixed shift) */ ++struct tnum tnum_lshift(struct tnum a, u8 shift); ++/* Shift (rsh) a tnum right (by a fixed shift) */ ++struct tnum tnum_rshift(struct tnum a, u8 shift); ++/* Shift (arsh) a tnum right (by a fixed min_shift) */ ++struct tnum tnum_arshift(struct tnum a, u8 min_shift, u8 insn_bitness); ++/* Add two tnums, return @a + @b */ ++struct tnum tnum_add(struct tnum a, struct tnum b); ++/* Subtract two tnums, return @a - @b */ ++struct tnum tnum_sub(struct tnum a, struct tnum b); ++/* Bitwise-AND, return @a & @b */ ++struct tnum tnum_and(struct tnum a, struct tnum b); ++/* Bitwise-OR, return @a | @b */ ++struct tnum tnum_or(struct tnum a, struct tnum b); ++/* Bitwise-XOR, return @a ^ @b */ ++struct tnum tnum_xor(struct tnum a, struct tnum b); ++/* Multiply two tnums, return @a * @b */ ++struct tnum tnum_mul(struct tnum a, struct tnum b); ++ ++/* Return a tnum representing numbers satisfying both @a and @b */ ++struct tnum tnum_intersect(struct tnum a, struct tnum b); ++ ++/* Return @a with all but the lowest @size bytes cleared */ ++struct tnum tnum_cast(struct tnum a, u8 size); ++ ++/* Returns true if @a is a known constant */ ++static inline bool tnum_is_const(struct tnum a) ++{ ++ return !a.mask; ++} ++ ++/* Returns true if @a == tnum_const(@b) */ ++static inline bool tnum_equals_const(struct tnum a, u64 b) ++{ ++ return tnum_is_const(a) && a.value == b; ++} ++ ++/* Returns true if @a is completely unknown */ ++static inline bool tnum_is_unknown(struct tnum a) ++{ ++ return !~a.mask; ++} ++ ++/* Returns true if @a is known to be a multiple of @size. ++ * @size must be a power of two. ++ */ ++bool tnum_is_aligned(struct tnum a, u64 size); ++ ++/* Returns true if @b represents a subset of @a. */ ++bool tnum_in(struct tnum a, struct tnum b); ++ ++/* Formatting functions. These have snprintf-like semantics: they will write ++ * up to @size bytes (including the terminating NUL byte), and return the number ++ * of bytes (excluding the terminating NUL) which would have been written had ++ * sufficient space been available. (Thus tnum_sbin always returns 64.) ++ */ ++/* Format a tnum as a pair of hex numbers (value; mask) */ ++int tnum_strn(char *str, size_t size, struct tnum a); ++/* Format a tnum as tristate binary expansion */ ++int tnum_sbin(char *str, size_t size, struct tnum a); ++ ++#endif /* _LINUX_TNUM_H */ +--- a/include/linux/bitmap.h ++++ b/include/linux/bitmap.h +@@ -326,6 +326,24 @@ static inline int bitmap_parse(const cha + return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); + } + ++/** ++ * bitmap_from_u64 - Check and swap words within u64. ++ * @mask: source bitmap ++ * @dst: destination bitmap ++ * ++ * In 32-bit Big Endian kernel, when using ``(u32 *)(&val)[*]`` ++ * to read u64 mask, we will get the wrong word. ++ * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits, ++ * but we expect the lower 32-bits of u64. ++ */ ++static inline void bitmap_from_u64(unsigned long *dst, u64 mask) ++{ ++ dst[0] = mask & ULONG_MAX; ++ ++ if (sizeof(mask) > sizeof(unsigned long)) ++ dst[1] = mask >> 32; ++} ++ + #endif /* __ASSEMBLY__ */ + + #endif /* __LINUX_BITMAP_H */ +--- /dev/null ++++ b/include/linux/overflow.h +@@ -0,0 +1,320 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR MIT */ ++#ifndef __LINUX_OVERFLOW_H ++#define __LINUX_OVERFLOW_H ++ ++#include ++#include ++ ++/* ++ * In the fallback code below, we need to compute the minimum and ++ * maximum values representable in a given type. These macros may also ++ * be useful elsewhere, so we provide them outside the ++ * COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW block. ++ * ++ * It would seem more obvious to do something like ++ * ++ * #define type_min(T) (T)(is_signed_type(T) ? (T)1 << (8*sizeof(T)-1) : 0) ++ * #define type_max(T) (T)(is_signed_type(T) ? ((T)1 << (8*sizeof(T)-1)) - 1 : ~(T)0) ++ * ++ * Unfortunately, the middle expressions, strictly speaking, have ++ * undefined behaviour, and at least some versions of gcc warn about ++ * the type_max expression (but not if -fsanitize=undefined is in ++ * effect; in that case, the warning is deferred to runtime...). ++ * ++ * The slightly excessive casting in type_min is to make sure the ++ * macros also produce sensible values for the exotic type _Bool. [The ++ * overflow checkers only almost work for _Bool, but that's ++ * a-feature-not-a-bug, since people shouldn't be doing arithmetic on ++ * _Bools. Besides, the gcc builtins don't allow _Bool* as third ++ * argument.] ++ * ++ * Idea stolen from ++ * https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html - ++ * credit to Christian Biere. ++ */ ++#define is_signed_type(type) (((type)(-1)) < (type)1) ++#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type))) ++#define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) ++#define type_min(T) ((T)((T)-type_max(T)-(T)1)) ++ ++/* ++ * Avoids triggering -Wtype-limits compilation warning, ++ * while using unsigned data types to check a < 0. ++ */ ++#define is_non_negative(a) ((a) > 0 || (a) == 0) ++#define is_negative(a) (!(is_non_negative(a))) ++ ++#ifdef COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW ++/* ++ * For simplicity and code hygiene, the fallback code below insists on ++ * a, b and *d having the same type (similar to the min() and max() ++ * macros), whereas gcc's type-generic overflow checkers accept ++ * different types. Hence we don't just make check_add_overflow an ++ * alias for __builtin_add_overflow, but add type checks similar to ++ * below. ++ */ ++#define check_add_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ __builtin_add_overflow(__a, __b, __d); \ ++}) ++ ++#define check_sub_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ __builtin_sub_overflow(__a, __b, __d); \ ++}) ++ ++#define check_mul_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ __builtin_mul_overflow(__a, __b, __d); \ ++}) ++ ++#else ++ ++ ++/* Checking for unsigned overflow is relatively easy without causing UB. */ ++#define __unsigned_add_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = __a + __b; \ ++ *__d < __a; \ ++}) ++#define __unsigned_sub_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = __a - __b; \ ++ __a < __b; \ ++}) ++/* ++ * If one of a or b is a compile-time constant, this avoids a division. ++ */ ++#define __unsigned_mul_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = __a * __b; \ ++ __builtin_constant_p(__b) ? \ ++ __b > 0 && __a > type_max(typeof(__a)) / __b : \ ++ __a > 0 && __b > type_max(typeof(__b)) / __a; \ ++}) ++ ++/* ++ * For signed types, detecting overflow is much harder, especially if ++ * we want to avoid UB. But the interface of these macros is such that ++ * we must provide a result in *d, and in fact we must produce the ++ * result promised by gcc's builtins, which is simply the possibly ++ * wrapped-around value. Fortunately, we can just formally do the ++ * operations in the widest relevant unsigned type (u64) and then ++ * truncate the result - gcc is smart enough to generate the same code ++ * with and without the (u64) casts. ++ */ ++ ++/* ++ * Adding two signed integers can overflow only if they have the same ++ * sign, and overflow has happened iff the result has the opposite ++ * sign. ++ */ ++#define __signed_add_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = (u64)__a + (u64)__b; \ ++ (((~(__a ^ __b)) & (*__d ^ __a)) \ ++ & type_min(typeof(__a))) != 0; \ ++}) ++ ++/* ++ * Subtraction is similar, except that overflow can now happen only ++ * when the signs are opposite. In this case, overflow has happened if ++ * the result has the opposite sign of a. ++ */ ++#define __signed_sub_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = (u64)__a - (u64)__b; \ ++ ((((__a ^ __b)) & (*__d ^ __a)) \ ++ & type_min(typeof(__a))) != 0; \ ++}) ++ ++/* ++ * Signed multiplication is rather hard. gcc always follows C99, so ++ * division is truncated towards 0. This means that we can write the ++ * overflow check like this: ++ * ++ * (a > 0 && (b > MAX/a || b < MIN/a)) || ++ * (a < -1 && (b > MIN/a || b < MAX/a) || ++ * (a == -1 && b == MIN) ++ * ++ * The redundant casts of -1 are to silence an annoying -Wtype-limits ++ * (included in -Wextra) warning: When the type is u8 or u16, the ++ * __b_c_e in check_mul_overflow obviously selects ++ * __unsigned_mul_overflow, but unfortunately gcc still parses this ++ * code and warns about the limited range of __b. ++ */ ++ ++#define __signed_mul_overflow(a, b, d) ({ \ ++ typeof(a) __a = (a); \ ++ typeof(b) __b = (b); \ ++ typeof(d) __d = (d); \ ++ typeof(a) __tmax = type_max(typeof(a)); \ ++ typeof(a) __tmin = type_min(typeof(a)); \ ++ (void) (&__a == &__b); \ ++ (void) (&__a == __d); \ ++ *__d = (u64)__a * (u64)__b; \ ++ (__b > 0 && (__a > __tmax/__b || __a < __tmin/__b)) || \ ++ (__b < (typeof(__b))-1 && (__a > __tmin/__b || __a < __tmax/__b)) || \ ++ (__b == (typeof(__b))-1 && __a == __tmin); \ ++}) ++ ++ ++#define check_add_overflow(a, b, d) \ ++ __builtin_choose_expr(is_signed_type(typeof(a)), \ ++ __signed_add_overflow(a, b, d), \ ++ __unsigned_add_overflow(a, b, d)) ++ ++#define check_sub_overflow(a, b, d) \ ++ __builtin_choose_expr(is_signed_type(typeof(a)), \ ++ __signed_sub_overflow(a, b, d), \ ++ __unsigned_sub_overflow(a, b, d)) ++ ++#define check_mul_overflow(a, b, d) \ ++ __builtin_choose_expr(is_signed_type(typeof(a)), \ ++ __signed_mul_overflow(a, b, d), \ ++ __unsigned_mul_overflow(a, b, d)) ++ ++ ++#endif /* COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW */ ++ ++/** check_shl_overflow() - Calculate a left-shifted value and check overflow ++ * ++ * @a: Value to be shifted ++ * @s: How many bits left to shift ++ * @d: Pointer to where to store the result ++ * ++ * Computes *@d = (@a << @s) ++ * ++ * Returns true if '*d' cannot hold the result or when 'a << s' doesn't ++ * make sense. Example conditions: ++ * - 'a << s' causes bits to be lost when stored in *d. ++ * - 's' is garbage (e.g. negative) or so large that the result of ++ * 'a << s' is guaranteed to be 0. ++ * - 'a' is negative. ++ * - 'a << s' sets the sign bit, if any, in '*d'. ++ * ++ * '*d' will hold the results of the attempted shift, but is not ++ * considered "safe for use" if false is returned. ++ */ ++#define check_shl_overflow(a, s, d) ({ \ ++ typeof(a) _a = a; \ ++ typeof(s) _s = s; \ ++ typeof(d) _d = d; \ ++ u64 _a_full = _a; \ ++ unsigned int _to_shift = \ ++ is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0; \ ++ *_d = (_a_full << _to_shift); \ ++ (_to_shift != _s || is_negative(*_d) || is_negative(_a) || \ ++ (*_d >> _to_shift) != _a); \ ++}) ++ ++/** ++ * array_size() - Calculate size of 2-dimensional array. ++ * ++ * @a: dimension one ++ * @b: dimension two ++ * ++ * Calculates size of 2-dimensional array: @a * @b. ++ * ++ * Returns: number of bytes needed to represent the array or SIZE_MAX on ++ * overflow. ++ */ ++static inline __must_check size_t array_size(size_t a, size_t b) ++{ ++ size_t bytes; ++ ++ if (check_mul_overflow(a, b, &bytes)) ++ return SIZE_MAX; ++ ++ return bytes; ++} ++ ++/** ++ * array3_size() - Calculate size of 3-dimensional array. ++ * ++ * @a: dimension one ++ * @b: dimension two ++ * @c: dimension three ++ * ++ * Calculates size of 3-dimensional array: @a * @b * @c. ++ * ++ * Returns: number of bytes needed to represent the array or SIZE_MAX on ++ * overflow. ++ */ ++static inline __must_check size_t array3_size(size_t a, size_t b, size_t c) ++{ ++ size_t bytes; ++ ++ if (check_mul_overflow(a, b, &bytes)) ++ return SIZE_MAX; ++ if (check_mul_overflow(bytes, c, &bytes)) ++ return SIZE_MAX; ++ ++ return bytes; ++} ++ ++/* ++ * Compute a*b+c, returning SIZE_MAX on overflow. Internal helper for ++ * struct_size() below. ++ */ ++static inline __must_check size_t __ab_c_size(size_t a, size_t b, size_t c) ++{ ++ size_t bytes; ++ ++ if (check_mul_overflow(a, b, &bytes)) ++ return SIZE_MAX; ++ if (check_add_overflow(bytes, c, &bytes)) ++ return SIZE_MAX; ++ ++ return bytes; ++} ++ ++/** ++ * struct_size() - Calculate size of structure with trailing array. ++ * @p: Pointer to the structure. ++ * @member: Name of the array member. ++ * @n: Number of elements in the array. ++ * ++ * Calculates size of memory needed for structure @p followed by an ++ * array of @n @member elements. ++ * ++ * Return: number of bytes needed or SIZE_MAX on overflow. ++ */ ++#define struct_size(p, member, n) \ ++ __ab_c_size(n, \ ++ sizeof(*(p)->member) + __must_be_array((p)->member),\ ++ sizeof(*(p))) ++ ++#endif /* __LINUX_OVERFLOW_H */ +--- a/net/core/filter.c ++++ b/net/core/filter.c +@@ -1,3 +1,4 @@ ++// SPDX-License-Identifier: GPL-2.0-or-later + /* + * Linux Socket Filter - Kernel level socket filtering + * +@@ -12,11 +13,6 @@ + * Alexei Starovoitov + * Daniel Borkmann + * +- * This program is free software; you can redistribute it and/or +- * modify it under the terms of the GNU General Public License +- * as published by the Free Software Foundation; either version +- * 2 of the License, or (at your option) any later version. +- * + * Andi Kleen - Fix a few bad bugs and races. + * Kris Katterjohn - Added many additional checks in bpf_check_classic() + */ +@@ -26,11 +22,14 @@ + #include + #include + #include ++#include + #include + #include + #include + #include ++#include + #include ++#include + #include + #include + #include +@@ -39,17 +38,32 @@ + #include + #include + #include +-#include ++#include + #include ++#include + #include + #include + #include + #include + #include + #include +-#include + #include + #include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + + /** + * sk_filter_trim_cap - run a packet through a socket filter +@@ -84,7 +98,12 @@ int sk_filter_trim_cap(struct sock *sk, + rcu_read_lock(); + filter = rcu_dereference(sk->sk_filter); + if (filter) { +- unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb); ++ struct sock *save_sk = skb->sk; ++ unsigned int pkt_len; ++ ++ skb->sk = sk; ++ pkt_len = bpf_prog_run_save_cb(filter->prog, skb); ++ skb->sk = save_sk; + err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM; + } + rcu_read_unlock(); +@@ -93,14 +112,13 @@ int sk_filter_trim_cap(struct sock *sk, + } + EXPORT_SYMBOL(sk_filter_trim_cap); + +-static u64 __skb_get_pay_offset(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) ++BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb) + { +- return skb_get_poff((struct sk_buff *)(unsigned long) ctx); ++ return skb_get_poff(skb); + } + +-static u64 __skb_get_nlattr(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) ++BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x) + { +- struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; + struct nlattr *nla; + + if (skb_is_nonlinear(skb)) +@@ -119,9 +137,8 @@ static u64 __skb_get_nlattr(u64 ctx, u64 + return 0; + } + +-static u64 __skb_get_nlattr_nest(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) ++BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x) + { +- struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; + struct nlattr *nla; + + if (skb_is_nonlinear(skb)) +@@ -144,11 +161,98 @@ static u64 __skb_get_nlattr_nest(u64 ctx + return 0; + } + +-static u64 __get_raw_cpu_id(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) ++BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *, ++ data, int, headlen, int, offset) ++{ ++ u8 tmp, *ptr; ++ const int len = sizeof(tmp); ++ ++ if (offset >= 0) { ++ if (headlen - offset >= len) ++ return *(u8 *)(data + offset); ++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) ++ return tmp; ++ } else { ++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); ++ if (likely(ptr)) ++ return *(u8 *)ptr; ++ } ++ ++ return -EFAULT; ++} ++ ++BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb, ++ int, offset) ++{ ++ return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len, ++ offset); ++} ++ ++BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *, ++ data, int, headlen, int, offset) ++{ ++ u16 tmp, *ptr; ++ const int len = sizeof(tmp); ++ ++ if (offset >= 0) { ++ if (headlen - offset >= len) ++ return get_unaligned_be16(data + offset); ++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) ++ return be16_to_cpu(tmp); ++ } else { ++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); ++ if (likely(ptr)) ++ return get_unaligned_be16(ptr); ++ } ++ ++ return -EFAULT; ++} ++ ++BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb, ++ int, offset) ++{ ++ return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len, ++ offset); ++} ++ ++BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *, ++ data, int, headlen, int, offset) ++{ ++ u32 tmp, *ptr; ++ const int len = sizeof(tmp); ++ ++ if (likely(offset >= 0)) { ++ if (headlen - offset >= len) ++ return get_unaligned_be32(data + offset); ++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) ++ return be32_to_cpu(tmp); ++ } else { ++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); ++ if (likely(ptr)) ++ return get_unaligned_be32(ptr); ++ } ++ ++ return -EFAULT; ++} ++ ++BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb, ++ int, offset) ++{ ++ return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len, ++ offset); ++} ++ ++BPF_CALL_0(bpf_get_raw_cpu_id) + { + return raw_smp_processor_id(); + } + ++static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { ++ .func = bpf_get_raw_cpu_id, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++}; ++ + static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, + struct bpf_insn *insn_buf) + { +@@ -178,22 +282,18 @@ static u32 convert_skb_access(int skb_fi + break; + + case SKF_AD_VLAN_TAG: +- case SKF_AD_VLAN_TAG_PRESENT: + BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); +- BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); + + /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */ + *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, + offsetof(struct sk_buff, vlan_tci)); +- if (skb_field == SKF_AD_VLAN_TAG) { +- *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, +- ~VLAN_TAG_PRESENT); +- } else { +- /* dst_reg >>= 12 */ +- *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 12); +- /* dst_reg &= 1 */ ++ break; ++ case SKF_AD_VLAN_TAG_PRESENT: ++ *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET()); ++ if (PKT_VLAN_PRESENT_BIT) ++ *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT); ++ if (PKT_VLAN_PRESENT_BIT < 7) + *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 1); +- } + break; + } + +@@ -226,9 +326,8 @@ static bool convert_bpf_extensions(struc + case SKF_AD_OFF + SKF_AD_HATYPE: + BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); + BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); +- BUILD_BUG_ON(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)) < 0); + +- *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), + BPF_REG_TMP, BPF_REG_CTX, + offsetof(struct sk_buff, dev)); + /* if (tmp != 0) goto pc + 1 */ +@@ -295,16 +394,16 @@ static bool convert_bpf_extensions(struc + /* Emit call(arg1=CTX, arg2=A, arg3=X) */ + switch (fp->k) { + case SKF_AD_OFF + SKF_AD_PAY_OFFSET: +- *insn = BPF_EMIT_CALL(__skb_get_pay_offset); ++ *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset); + break; + case SKF_AD_OFF + SKF_AD_NLATTR: +- *insn = BPF_EMIT_CALL(__skb_get_nlattr); ++ *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr); + break; + case SKF_AD_OFF + SKF_AD_NLATTR_NEST: +- *insn = BPF_EMIT_CALL(__skb_get_nlattr_nest); ++ *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest); + break; + case SKF_AD_OFF + SKF_AD_CPU: +- *insn = BPF_EMIT_CALL(__get_raw_cpu_id); ++ *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id); + break; + case SKF_AD_OFF + SKF_AD_RANDOM: + *insn = BPF_EMIT_CALL(bpf_user_rnd_u32); +@@ -331,35 +430,101 @@ static bool convert_bpf_extensions(struc + return true; + } + ++static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp) ++{ ++ const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS); ++ int size = bpf_size_to_bytes(BPF_SIZE(fp->code)); ++ bool endian = BPF_SIZE(fp->code) == BPF_H || ++ BPF_SIZE(fp->code) == BPF_W; ++ bool indirect = BPF_MODE(fp->code) == BPF_IND; ++ const int ip_align = NET_IP_ALIGN; ++ struct bpf_insn *insn = *insnp; ++ int offset = fp->k; ++ ++ if (!indirect && ++ ((unaligned_ok && offset >= 0) || ++ (!unaligned_ok && offset >= 0 && ++ offset + ip_align >= 0 && ++ offset + ip_align % size == 0))) { ++ bool ldx_off_ok = offset <= S16_MAX; ++ ++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H); ++ if (offset) ++ *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset); ++ *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP, ++ size, 2 + endian + (!ldx_off_ok * 2)); ++ if (ldx_off_ok) { ++ *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, ++ BPF_REG_D, offset); ++ } else { ++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D); ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset); ++ *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, ++ BPF_REG_TMP, 0); ++ } ++ if (endian) ++ *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8); ++ *insn++ = BPF_JMP_A(8); ++ } ++ ++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX); ++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D); ++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H); ++ if (!indirect) { ++ *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset); ++ } else { ++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X); ++ if (fp->k) ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset); ++ } ++ ++ switch (BPF_SIZE(fp->code)) { ++ case BPF_B: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8); ++ break; ++ case BPF_H: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16); ++ break; ++ case BPF_W: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32); ++ break; ++ default: ++ return false; ++ } ++ ++ *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2); ++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); ++ *insn = BPF_EXIT_INSN(); ++ ++ *insnp = insn; ++ return true; ++} ++ + /** + * bpf_convert_filter - convert filter program + * @prog: the user passed filter program + * @len: the length of the user passed filter program +- * @new_prog: buffer where converted program will be stored ++ * @new_prog: allocated 'struct bpf_prog' or NULL + * @new_len: pointer to store length of converted program ++ * @seen_ld_abs: bool whether we've seen ld_abs/ind + * +- * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style. ++ * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn' ++ * style extended BPF (eBPF). + * Conversion workflow: + * + * 1) First pass for calculating the new program length: +- * bpf_convert_filter(old_prog, old_len, NULL, &new_len) ++ * bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs) + * + * 2) 2nd pass to remap in two passes: 1st pass finds new + * jump offsets, 2nd pass remapping: +- * new_prog = kmalloc(sizeof(struct bpf_insn) * new_len); +- * bpf_convert_filter(old_prog, old_len, new_prog, &new_len); +- * +- * User BPF's register A is mapped to our BPF register 6, user BPF +- * register X is mapped to BPF register 7; frame pointer is always +- * register 10; Context 'void *ctx' is stored in register 1, that is, +- * for socket filters: ctx == 'struct sk_buff *', for seccomp: +- * ctx == 'struct seccomp_data *'. ++ * bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs) + */ + static int bpf_convert_filter(struct sock_filter *prog, int len, +- struct bpf_insn *new_prog, int *new_len) ++ struct bpf_prog *new_prog, int *new_len, ++ bool *seen_ld_abs) + { +- int new_flen = 0, pass = 0, target, i; +- struct bpf_insn *new_insn; ++ int new_flen = 0, pass = 0, target, i, stack_off; ++ struct bpf_insn *new_insn, *first_insn = NULL; + struct sock_filter *fp; + int *addrs = NULL; + u8 bpf_src; +@@ -371,6 +536,7 @@ static int bpf_convert_filter(struct soc + return -EINVAL; + + if (new_prog) { ++ first_insn = new_prog->insnsi; + addrs = kcalloc(len, sizeof(*addrs), + GFP_KERNEL | __GFP_NOWARN); + if (!addrs) +@@ -378,19 +544,47 @@ static int bpf_convert_filter(struct soc + } + + do_pass: +- new_insn = new_prog; ++ new_insn = first_insn; + fp = prog; + +- if (new_insn) +- *new_insn = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); +- new_insn++; ++ /* Classic BPF related prologue emission. */ ++ if (new_prog) { ++ /* Classic BPF expects A and X to be reset first. These need ++ * to be guaranteed to be the first two instructions. ++ */ ++ *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); ++ *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X); ++ ++ /* All programs must keep CTX in callee saved BPF_REG_CTX. ++ * In eBPF case it's done by the compiler, here we need to ++ * do this ourself. Initial CTX is present in BPF_REG_ARG1. ++ */ ++ *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); ++ if (*seen_ld_abs) { ++ /* For packet access in classic BPF, cache skb->data ++ * in callee-saved BPF R8 and skb->len - skb->data_len ++ * (headlen) in BPF R9. Since classic BPF is read-only ++ * on CTX, we only need to cache it once. ++ */ ++ *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), ++ BPF_REG_D, BPF_REG_CTX, ++ offsetof(struct sk_buff, data)); ++ *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX, ++ offsetof(struct sk_buff, len)); ++ *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX, ++ offsetof(struct sk_buff, data_len)); ++ *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP); ++ } ++ } else { ++ new_insn += 3; ++ } + + for (i = 0; i < len; fp++, i++) { +- struct bpf_insn tmp_insns[6] = { }; ++ struct bpf_insn tmp_insns[32] = { }; + struct bpf_insn *insn = tmp_insns; + + if (addrs) +- addrs[i] = new_insn - new_prog; ++ addrs[i] = new_insn - first_insn; + + switch (fp->code) { + /* All arithmetic insns and skb loads map as-is. */ +@@ -429,6 +623,22 @@ do_pass: + BPF_MODE(fp->code) == BPF_ABS && + convert_bpf_extensions(fp, &insn)) + break; ++ if (BPF_CLASS(fp->code) == BPF_LD && ++ convert_bpf_ld_abs(fp, &insn)) { ++ *seen_ld_abs = true; ++ break; ++ } ++ ++ if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) || ++ fp->code == (BPF_ALU | BPF_MOD | BPF_X)) { ++ *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X); ++ /* Error with exception code on div/mod by 0. ++ * For cBPF programs, this was always return 0. ++ */ ++ *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2); ++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); ++ *insn++ = BPF_EXIT_INSN(); ++ } + + *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k); + break; +@@ -441,11 +651,18 @@ do_pass: + + #define BPF_EMIT_JMP \ + do { \ ++ const s32 off_min = S16_MIN, off_max = S16_MAX; \ ++ s32 off; \ ++ \ + if (target >= len || target < 0) \ + goto err; \ +- insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ ++ off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ + /* Adjust pc relative offset for 2nd or 3rd insn. */ \ +- insn->off -= insn - tmp_insns; \ ++ off -= insn - tmp_insns; \ ++ /* Reject anything not fitting into insn->off. */ \ ++ if (off < off_min || off > off_max) \ ++ goto err; \ ++ insn->off = off; \ + } while (0) + + case BPF_JMP | BPF_JA: +@@ -487,14 +704,27 @@ do_pass: + break; + } + +- /* Convert JEQ into JNE when 'jump_true' is next insn. */ +- if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) { +- insn->code = BPF_JMP | BPF_JNE | bpf_src; ++ /* Convert some jumps when 'jump_true' is next insn. */ ++ if (fp->jt == 0) { ++ switch (BPF_OP(fp->code)) { ++ case BPF_JEQ: ++ insn->code = BPF_JMP | BPF_JNE | bpf_src; ++ break; ++ case BPF_JGT: ++ insn->code = BPF_JMP | BPF_JLE | bpf_src; ++ break; ++ case BPF_JGE: ++ insn->code = BPF_JMP | BPF_JLT | bpf_src; ++ break; ++ default: ++ goto jmp_rest; ++ } ++ + target = i + fp->jf + 1; + BPF_EMIT_JMP; + break; + } +- ++jmp_rest: + /* Other jumps are mapped into two insns: Jxx and JA. */ + target = i + fp->jt + 1; + insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; +@@ -507,44 +737,64 @@ do_pass: + break; + + /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */ +- case BPF_LDX | BPF_MSH | BPF_B: +- /* tmp = A */ +- *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_A); ++ case BPF_LDX | BPF_MSH | BPF_B: { ++ struct sock_filter tmp = { ++ .code = BPF_LD | BPF_ABS | BPF_B, ++ .k = fp->k, ++ }; ++ ++ *seen_ld_abs = true; ++ ++ /* X = A */ ++ *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); + /* A = BPF_R0 = *(u8 *) (skb->data + K) */ +- *insn++ = BPF_LD_ABS(BPF_B, fp->k); ++ convert_bpf_ld_abs(&tmp, &insn); ++ insn++; + /* A &= 0xf */ + *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf); + /* A <<= 2 */ + *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2); ++ /* tmp = X */ ++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X); + /* X = A */ + *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); + /* A = tmp */ + *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP); + break; +- +- /* RET_K, RET_A are remaped into 2 insns. */ ++ } ++ /* RET_K is remaped into 2 insns. RET_A case doesn't need an ++ * extra mov as BPF_REG_0 is already mapped into BPF_REG_A. ++ */ + case BPF_RET | BPF_A: + case BPF_RET | BPF_K: +- *insn++ = BPF_MOV32_RAW(BPF_RVAL(fp->code) == BPF_K ? +- BPF_K : BPF_X, BPF_REG_0, +- BPF_REG_A, fp->k); ++ if (BPF_RVAL(fp->code) == BPF_K) ++ *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0, ++ 0, fp->k); + *insn = BPF_EXIT_INSN(); + break; + + /* Store to stack. */ + case BPF_ST: + case BPF_STX: ++ stack_off = fp->k * 4 + 4; + *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) == + BPF_ST ? BPF_REG_A : BPF_REG_X, +- -(BPF_MEMWORDS - fp->k) * 4); ++ -stack_off); ++ /* check_load_and_stores() verifies that classic BPF can ++ * load from stack only after write, so tracking ++ * stack_depth for ST|STX insns is enough ++ */ ++ if (new_prog && new_prog->aux->stack_depth < stack_off) ++ new_prog->aux->stack_depth = stack_off; + break; + + /* Load from stack. */ + case BPF_LD | BPF_MEM: + case BPF_LDX | BPF_MEM: ++ stack_off = fp->k * 4 + 4; + *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? + BPF_REG_A : BPF_REG_X, BPF_REG_FP, +- -(BPF_MEMWORDS - fp->k) * 4); ++ -stack_off); + break; + + /* A = K or X = K */ +@@ -592,13 +842,15 @@ do_pass: + + if (!new_prog) { + /* Only calculating new length. */ +- *new_len = new_insn - new_prog; ++ *new_len = new_insn - first_insn; ++ if (*seen_ld_abs) ++ *new_len += 4; /* Prologue bits. */ + return 0; + } + + pass++; +- if (new_flen != new_insn - new_prog) { +- new_flen = new_insn - new_prog; ++ if (new_flen != new_insn - first_insn) { ++ new_flen = new_insn - first_insn; + if (pass > 2) + goto err; + goto do_pass; +@@ -738,6 +990,17 @@ static bool chk_code_allowed(u16 code_to + return codes[code_to_probe]; + } + ++static bool bpf_check_basics_ok(const struct sock_filter *filter, ++ unsigned int flen) ++{ ++ if (filter == NULL) ++ return false; ++ if (flen == 0 || flen > BPF_MAXINSNS) ++ return false; ++ ++ return true; ++} ++ + /** + * bpf_check_classic - verify socket filter code + * @filter: filter to verify +@@ -758,9 +1021,6 @@ static int bpf_check_classic(const struc + bool anc_found; + int pc; + +- if (flen == 0 || flen > BPF_MAXINSNS) +- return -EINVAL; +- + /* Check the filter code now */ + for (pc = 0; pc < flen; pc++) { + const struct sock_filter *ftest = &filter[pc]; +@@ -901,7 +1161,7 @@ static void sk_filter_release_rcu(struct + */ + static void sk_filter_release(struct sk_filter *fp) + { +- if (atomic_dec_and_test(&fp->refcnt)) ++ if (refcount_dec_and_test(&fp->refcnt)) + call_rcu(&fp->rcu, sk_filter_release_rcu); + } + +@@ -916,25 +1176,37 @@ void sk_filter_uncharge(struct sock *sk, + /* try to charge the socket memory if there is space available + * return true on success + */ +-bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) ++static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp) + { + u32 filter_size = bpf_prog_size(fp->prog->len); + + /* same check as in sock_kmalloc() */ + if (filter_size <= sysctl_optmem_max && + atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) { +- atomic_inc(&fp->refcnt); + atomic_add(filter_size, &sk->sk_omem_alloc); + return true; + } + return false; + } + ++bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) ++{ ++ if (!refcount_inc_not_zero(&fp->refcnt)) ++ return false; ++ ++ if (!__sk_filter_charge(sk, fp)) { ++ sk_filter_release(fp); ++ return false; ++ } ++ return true; ++} ++ + static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp) + { + struct sock_filter *old_prog; + struct bpf_prog *old_fp; + int err, new_len, old_len = fp->len; ++ bool seen_ld_abs = false; + + /* We are free to overwrite insns et al right here as it + * won't be used at this point in time anymore internally +@@ -956,7 +1228,8 @@ static struct bpf_prog *bpf_migrate_filt + } + + /* 1st pass: calculate the new program length. */ +- err = bpf_convert_filter(old_prog, old_len, NULL, &new_len); ++ err = bpf_convert_filter(old_prog, old_len, NULL, &new_len, ++ &seen_ld_abs); + if (err) + goto out_err_free; + +@@ -975,7 +1248,8 @@ static struct bpf_prog *bpf_migrate_filt + fp->len = new_len; + + /* 2nd pass: remap sock_filter insns into bpf_insn insns. */ +- err = bpf_convert_filter(old_prog, old_len, fp->insnsi, &new_len); ++ err = bpf_convert_filter(old_prog, old_len, fp, &new_len, ++ &seen_ld_abs); + if (err) + /* 2nd bpf_convert_filter() can fail only if it fails + * to allocate memory, remapping must succeed. Note, +@@ -984,7 +1258,9 @@ static struct bpf_prog *bpf_migrate_filt + */ + goto out_err_free; + +- bpf_prog_select_runtime(fp); ++ fp = bpf_prog_select_runtime(fp, &err); ++ if (err) ++ goto out_err_free; + + kfree(old_prog); + return fp; +@@ -1051,7 +1327,7 @@ int bpf_prog_create(struct bpf_prog **pf + struct bpf_prog *fp; + + /* Make sure new filter is there and in the right amounts. */ +- if (fprog->filter == NULL) ++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) + return -EINVAL; + + fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); +@@ -1098,7 +1374,7 @@ int bpf_prog_create_from_user(struct bpf + int err; + + /* Make sure new filter is there and in the right amounts. */ +- if (fprog->filter == NULL) ++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) + return -EINVAL; + + fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); +@@ -1139,8 +1415,7 @@ void bpf_prog_destroy(struct bpf_prog *f + } + EXPORT_SYMBOL_GPL(bpf_prog_destroy); + +-static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk, +- bool locked) ++static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk) + { + struct sk_filter *fp, *old_fp; + +@@ -1149,53 +1424,44 @@ static int __sk_attach_prog(struct bpf_p + return -ENOMEM; + + fp->prog = prog; +- atomic_set(&fp->refcnt, 0); + +- if (!sk_filter_charge(sk, fp)) { ++ if (!__sk_filter_charge(sk, fp)) { + kfree(fp); + return -ENOMEM; + } ++ refcount_set(&fp->refcnt, 1); + +- old_fp = rcu_dereference_protected(sk->sk_filter, locked); ++ old_fp = rcu_dereference_protected(sk->sk_filter, ++ lockdep_sock_is_held(sk)); + rcu_assign_pointer(sk->sk_filter, fp); ++ + if (old_fp) + sk_filter_uncharge(sk, old_fp); + + return 0; + } + +-/** +- * sk_attach_filter - attach a socket filter +- * @fprog: the filter program +- * @sk: the socket to use +- * +- * Attach the user's filter code. We first run some sanity checks on +- * it to make sure it does not explode on us later. If an error +- * occurs or there is insufficient memory for the filter a negative +- * errno code is returned. On success the return is zero. +- */ +-int __sk_attach_filter(struct sock_fprog *fprog, struct sock *sk, +- bool locked) ++static ++struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk) + { + unsigned int fsize = bpf_classic_proglen(fprog); +- unsigned int bpf_fsize = bpf_prog_size(fprog->len); + struct bpf_prog *prog; + int err; + + if (sock_flag(sk, SOCK_FILTER_LOCKED)) +- return -EPERM; ++ return ERR_PTR(-EPERM); + + /* Make sure new filter is there and in the right amounts. */ +- if (fprog->filter == NULL) +- return -EINVAL; ++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) ++ return ERR_PTR(-EINVAL); + +- prog = bpf_prog_alloc(bpf_fsize, 0); ++ prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); + if (!prog) +- return -ENOMEM; ++ return ERR_PTR(-ENOMEM); + + if (copy_from_user(prog->insns, fprog->filter, fsize)) { + __bpf_prog_free(prog); +- return -EFAULT; ++ return ERR_PTR(-EINVAL); + } + + prog->len = fprog->len; +@@ -1203,17 +1469,34 @@ int __sk_attach_filter(struct sock_fprog + err = bpf_prog_store_orig_filter(prog, fprog); + if (err) { + __bpf_prog_free(prog); +- return -ENOMEM; ++ return ERR_PTR(-ENOMEM); + } + + /* bpf_prepare_filter() already takes care of freeing + * memory in case something goes wrong. + */ +- prog = bpf_prepare_filter(prog, NULL); ++ return bpf_prepare_filter(prog, NULL); ++} ++ ++/** ++ * sk_attach_filter - attach a socket filter ++ * @fprog: the filter program ++ * @sk: the socket to use ++ * ++ * Attach the user's filter code. We first run some sanity checks on ++ * it to make sure it does not explode on us later. If an error ++ * occurs or there is insufficient memory for the filter a negative ++ * errno code is returned. On success the return is zero. ++ */ ++int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) ++{ ++ struct bpf_prog *prog = __get_filter(fprog, sk); ++ int err; ++ + if (IS_ERR(prog)) + return PTR_ERR(prog); + +- err = __sk_attach_prog(prog, sk, locked); ++ err = __sk_attach_prog(prog, sk); + if (err < 0) { + __bpf_prog_release(prog); + return err; +@@ -1221,31 +1504,25 @@ int __sk_attach_filter(struct sock_fprog + + return 0; + } +-EXPORT_SYMBOL_GPL(__sk_attach_filter); ++EXPORT_SYMBOL_GPL(sk_attach_filter); + +-int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) ++static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk) + { +- return __sk_attach_filter(fprog, sk, sock_owned_by_user(sk)); ++ if (sock_flag(sk, SOCK_FILTER_LOCKED)) ++ return ERR_PTR(-EPERM); ++ ++ return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER); + } + + int sk_attach_bpf(u32 ufd, struct sock *sk) + { +- struct bpf_prog *prog; ++ struct bpf_prog *prog = __get_bpf(ufd, sk); + int err; + +- if (sock_flag(sk, SOCK_FILTER_LOCKED)) +- return -EPERM; +- +- prog = bpf_prog_get(ufd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + +- if (prog->type != BPF_PROG_TYPE_SOCKET_FILTER) { +- bpf_prog_put(prog); +- return -EINVAL; +- } +- +- err = __sk_attach_prog(prog, sk, sock_owned_by_user(sk)); ++ err = __sk_attach_prog(prog, sk); + if (err < 0) { + bpf_prog_put(prog); + return err; +@@ -1254,79 +1531,201 @@ int sk_attach_bpf(u32 ufd, struct sock * + return 0; + } + +-#define BPF_RECOMPUTE_CSUM(flags) ((flags) & 1) ++struct bpf_scratchpad { ++ union { ++ __be32 diff[MAX_BPF_STACK / sizeof(__be32)]; ++ u8 buff[MAX_BPF_STACK]; ++ }; ++}; ++ ++static DEFINE_PER_CPU(struct bpf_scratchpad, bpf_sp); ++ ++static inline int __bpf_try_make_writable(struct sk_buff *skb, ++ unsigned int write_len) ++{ ++ return skb_ensure_writable(skb, write_len); ++} ++ ++static inline int bpf_try_make_writable(struct sk_buff *skb, ++ unsigned int write_len) ++{ ++ int err = __bpf_try_make_writable(skb, write_len); ++ ++ bpf_compute_data_pointers(skb); ++ return err; ++} ++ ++static int bpf_try_make_head_writable(struct sk_buff *skb) ++{ ++ return bpf_try_make_writable(skb, skb_headlen(skb)); ++} + +-static u64 bpf_skb_store_bytes(u64 r1, u64 r2, u64 r3, u64 r4, u64 flags) ++static inline void bpf_push_mac_rcsum(struct sk_buff *skb) ++{ ++ if (skb_at_tc_ingress(skb)) ++ skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len); ++} ++ ++static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) ++{ ++ if (skb_at_tc_ingress(skb)) ++ skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); ++} ++ ++BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, ++ const void *, from, u32, len, u64, flags) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- int offset = (int) r2; +- void *from = (void *) (long) r3; +- unsigned int len = (unsigned int) r4; +- char buf[16]; + void *ptr; + +- /* bpf verifier guarantees that: +- * 'from' pointer points to bpf program stack +- * 'len' bytes of it were initialized +- * 'len' > 0 +- * 'skb' is a valid pointer to 'struct sk_buff' +- * +- * so check for invalid 'offset' and too large 'len' +- */ +- if (unlikely((u32) offset > 0xffff || len > sizeof(buf))) ++ if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH))) ++ return -EINVAL; ++ if (unlikely(offset > 0xffff)) + return -EFAULT; +- if (unlikely(skb_try_make_writable(skb, offset + len))) ++ if (unlikely(bpf_try_make_writable(skb, offset + len))) + return -EFAULT; + +- ptr = skb_header_pointer(skb, offset, len, buf); +- if (unlikely(!ptr)) +- return -EFAULT; +- +- if (BPF_RECOMPUTE_CSUM(flags)) +- skb_postpull_rcsum(skb, ptr, len); ++ ptr = skb->data + offset; ++ if (flags & BPF_F_RECOMPUTE_CSUM) ++ __skb_postpull_rcsum(skb, ptr, len, offset); + + memcpy(ptr, from, len); + +- if (ptr == buf) +- /* skb_store_bits cannot return -EFAULT here */ +- skb_store_bits(skb, offset, ptr, len); ++ if (flags & BPF_F_RECOMPUTE_CSUM) ++ __skb_postpush_rcsum(skb, ptr, len, offset); ++ if (flags & BPF_F_INVALIDATE_HASH) ++ skb_clear_hash(skb); + +- if (BPF_RECOMPUTE_CSUM(flags) && skb->ip_summed == CHECKSUM_COMPLETE) +- skb->csum = csum_add(skb->csum, csum_partial(ptr, len, 0)); + return 0; + } + +-const struct bpf_func_proto bpf_skb_store_bytes_proto = { ++static const struct bpf_func_proto bpf_skb_store_bytes_proto = { + .func = bpf_skb_store_bytes, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, +- .arg3_type = ARG_PTR_TO_STACK, +- .arg4_type = ARG_CONST_STACK_SIZE, ++ .arg3_type = ARG_PTR_TO_MEM, ++ .arg4_type = ARG_CONST_SIZE, + .arg5_type = ARG_ANYTHING, + }; + +-#define BPF_HEADER_FIELD_SIZE(flags) ((flags) & 0x0f) +-#define BPF_IS_PSEUDO_HEADER(flags) ((flags) & 0x10) ++BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, ++ void *, to, u32, len) ++{ ++ void *ptr; ++ ++ if (unlikely(offset > 0xffff)) ++ goto err_clear; ++ ++ ptr = skb_header_pointer(skb, offset, len, to); ++ if (unlikely(!ptr)) ++ goto err_clear; ++ if (ptr != to) ++ memcpy(to, ptr, len); + +-static u64 bpf_l3_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) ++ return 0; ++err_clear: ++ memset(to, 0, len); ++ return -EFAULT; ++} ++ ++static const struct bpf_func_proto bpf_skb_load_bytes_proto = { ++ .func = bpf_skb_load_bytes, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg4_type = ARG_CONST_SIZE, ++}; ++ ++BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, ++ u32, offset, void *, to, u32, len, u32, start_header) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- int offset = (int) r2; +- __sum16 sum, *ptr; ++ u8 *end = skb_tail_pointer(skb); ++ u8 *start, *ptr; + +- if (unlikely((u32) offset > 0xffff)) +- return -EFAULT; ++ if (unlikely(offset > 0xffff)) ++ goto err_clear; + +- if (unlikely(skb_try_make_writable(skb, offset + sizeof(sum)))) +- return -EFAULT; ++ switch (start_header) { ++ case BPF_HDR_START_MAC: ++ if (unlikely(!skb_mac_header_was_set(skb))) ++ goto err_clear; ++ start = skb_mac_header(skb); ++ break; ++ case BPF_HDR_START_NET: ++ start = skb_network_header(skb); ++ break; ++ default: ++ goto err_clear; ++ } + +- ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); +- if (unlikely(!ptr)) ++ ptr = start + offset; ++ ++ if (likely(ptr + len <= end)) { ++ memcpy(to, ptr, len); ++ return 0; ++ } ++ ++err_clear: ++ memset(to, 0, len); ++ return -EFAULT; ++} ++ ++static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = { ++ .func = bpf_skb_load_bytes_relative, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg4_type = ARG_CONST_SIZE, ++ .arg5_type = ARG_ANYTHING, ++}; ++ ++BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len) ++{ ++ /* Idea is the following: should the needed direct read/write ++ * test fail during runtime, we can pull in more data and redo ++ * again, since implicitly, we invalidate previous checks here. ++ * ++ * Or, since we know how much we need to make read/writeable, ++ * this can be done once at the program beginning for direct ++ * access case. By this we overcome limitations of only current ++ * headroom being accessible. ++ */ ++ return bpf_try_make_writable(skb, len ? : skb_headlen(skb)); ++} ++ ++static const struct bpf_func_proto bpf_skb_pull_data_proto = { ++ .func = bpf_skb_pull_data, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++}; ++ ++BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset, ++ u64, from, u64, to, u64, flags) ++{ ++ __sum16 *ptr; ++ ++ if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK))) ++ return -EINVAL; ++ if (unlikely(offset > 0xffff || offset & 1)) ++ return -EFAULT; ++ if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) + return -EFAULT; + +- switch (BPF_HEADER_FIELD_SIZE(flags)) { ++ ptr = (__sum16 *)(skb->data + offset); ++ switch (flags & BPF_F_HDR_FIELD_MASK) { ++ case 0: ++ if (unlikely(from != 0)) ++ return -EINVAL; ++ ++ csum_replace_by_diff(ptr, to); ++ break; + case 2: + csum_replace2(ptr, from, to); + break; +@@ -1337,14 +1736,10 @@ static u64 bpf_l3_csum_replace(u64 r1, u + return -EINVAL; + } + +- if (ptr == &sum) +- /* skb_store_bits guaranteed to not return -EFAULT here */ +- skb_store_bits(skb, offset, ptr, sizeof(sum)); +- + return 0; + } + +-const struct bpf_func_proto bpf_l3_csum_replace_proto = { ++static const struct bpf_func_proto bpf_l3_csum_replace_proto = { + .func = bpf_l3_csum_replace, + .gpl_only = false, + .ret_type = RET_INTEGER, +@@ -1355,23 +1750,33 @@ const struct bpf_func_proto bpf_l3_csum_ + .arg5_type = ARG_ANYTHING, + }; + +-static u64 bpf_l4_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) ++BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset, ++ u64, from, u64, to, u64, flags) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- bool is_pseudo = !!BPF_IS_PSEUDO_HEADER(flags); +- int offset = (int) r2; +- __sum16 sum, *ptr; ++ bool is_pseudo = flags & BPF_F_PSEUDO_HDR; ++ bool is_mmzero = flags & BPF_F_MARK_MANGLED_0; ++ bool do_mforce = flags & BPF_F_MARK_ENFORCE; ++ __sum16 *ptr; + +- if (unlikely((u32) offset > 0xffff)) ++ if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE | ++ BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK))) ++ return -EINVAL; ++ if (unlikely(offset > 0xffff || offset & 1)) + return -EFAULT; +- if (unlikely(skb_try_make_writable(skb, offset + sizeof(sum)))) ++ if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) + return -EFAULT; + +- ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); +- if (unlikely(!ptr)) +- return -EFAULT; ++ ptr = (__sum16 *)(skb->data + offset); ++ if (is_mmzero && !do_mforce && !*ptr) ++ return 0; + +- switch (BPF_HEADER_FIELD_SIZE(flags)) { ++ switch (flags & BPF_F_HDR_FIELD_MASK) { ++ case 0: ++ if (unlikely(from != 0)) ++ return -EINVAL; ++ ++ inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo); ++ break; + case 2: + inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo); + break; +@@ -1382,14 +1787,12 @@ static u64 bpf_l4_csum_replace(u64 r1, u + return -EINVAL; + } + +- if (ptr == &sum) +- /* skb_store_bits guaranteed to not return -EFAULT here */ +- skb_store_bits(skb, offset, ptr, sizeof(sum)); +- ++ if (is_mmzero && !*ptr) ++ *ptr = CSUM_MANGLED_0; + return 0; + } + +-const struct bpf_func_proto bpf_l4_csum_replace_proto = { ++static const struct bpf_func_proto bpf_l4_csum_replace_proto = { + .func = bpf_l4_csum_replace, + .gpl_only = false, + .ret_type = RET_INTEGER, +@@ -1400,30 +1803,172 @@ const struct bpf_func_proto bpf_l4_csum_ + .arg5_type = ARG_ANYTHING, + }; + +-#define BPF_IS_REDIRECT_INGRESS(flags) ((flags) & 1) ++BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size, ++ __be32 *, to, u32, to_size, __wsum, seed) ++{ ++ struct bpf_scratchpad *sp = this_cpu_ptr(&bpf_sp); ++ u32 diff_size = from_size + to_size; ++ int i, j = 0; ++ ++ /* This is quite flexible, some examples: ++ * ++ * from_size == 0, to_size > 0, seed := csum --> pushing data ++ * from_size > 0, to_size == 0, seed := csum --> pulling data ++ * from_size > 0, to_size > 0, seed := 0 --> diffing data ++ * ++ * Even for diffing, from_size and to_size don't need to be equal. ++ */ ++ if (unlikely(((from_size | to_size) & (sizeof(__be32) - 1)) || ++ diff_size > sizeof(sp->diff))) ++ return -EINVAL; ++ ++ for (i = 0; i < from_size / sizeof(__be32); i++, j++) ++ sp->diff[j] = ~from[i]; ++ for (i = 0; i < to_size / sizeof(__be32); i++, j++) ++ sp->diff[j] = to[i]; ++ ++ return csum_partial(sp->diff, diff_size, seed); ++} ++ ++static const struct bpf_func_proto bpf_csum_diff_proto = { ++ .func = bpf_csum_diff, ++ .gpl_only = false, ++ .pkt_access = true, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_MEM_OR_NULL, ++ .arg2_type = ARG_CONST_SIZE_OR_ZERO, ++ .arg3_type = ARG_PTR_TO_MEM_OR_NULL, ++ .arg4_type = ARG_CONST_SIZE_OR_ZERO, ++ .arg5_type = ARG_ANYTHING, ++}; ++ ++BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum) ++{ ++ /* The interface is to be used in combination with bpf_csum_diff() ++ * for direct packet writes. csum rotation for alignment as well ++ * as emulating csum_sub() can be done from the eBPF program. ++ */ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ return (skb->csum = csum_add(skb->csum, csum)); ++ ++ return -ENOTSUPP; ++} ++ ++static const struct bpf_func_proto bpf_csum_update_proto = { ++ .func = bpf_csum_update, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++}; ++ ++static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ return dev_forward_skb(dev, skb); ++} ++ ++static inline int __bpf_rx_skb_no_mac(struct net_device *dev, ++ struct sk_buff *skb) ++{ ++ int ret = ____dev_forward_skb(dev, skb); ++ ++ if (likely(!ret)) { ++ skb->dev = dev; ++ ret = netif_rx(skb); ++ } ++ ++ return ret; ++} ++ ++static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ int ret; ++ ++ skb->dev = dev; ++ skb->tstamp.tv64 = 0; ++ ++ ret = dev_queue_xmit(skb); ++ ++ return ret; ++} ++ ++static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev, ++ u32 flags) ++{ ++ unsigned int mlen = skb_network_offset(skb); ++ ++ if (mlen) { ++ __skb_pull(skb, mlen); ++ ++ /* At ingress, the mac header has already been pulled once. ++ * At egress, skb_pospull_rcsum has to be done in case that ++ * the skb is originated from ingress (i.e. a forwarded skb) ++ * to ensure that rcsum starts at net header. ++ */ ++ if (!skb_at_tc_ingress(skb)) ++ skb_postpull_rcsum(skb, skb_mac_header(skb), mlen); ++ } ++ skb_pop_mac_header(skb); ++ skb_reset_mac_len(skb); ++ return flags & BPF_F_INGRESS ? ++ __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb); ++} + +-static u64 bpf_clone_redirect(u64 r1, u64 ifindex, u64 flags, u64 r4, u64 r5) ++static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev, ++ u32 flags) ++{ ++ /* Verify that a link layer header is carried */ ++ if (unlikely(skb->mac_header >= skb->network_header)) { ++ kfree_skb(skb); ++ return -ERANGE; ++ } ++ ++ bpf_push_mac_rcsum(skb); ++ return flags & BPF_F_INGRESS ? ++ __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb); ++} ++ ++static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev, ++ u32 flags) ++{ ++ if (dev_is_mac_header_xmit(dev)) ++ return __bpf_redirect_common(skb, dev, flags); ++ else ++ return __bpf_redirect_no_mac(skb, dev, flags); ++} ++ ++BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1, *skb2; + struct net_device *dev; ++ struct sk_buff *clone; ++ int ret; ++ ++ if (unlikely(flags & ~(BPF_F_INGRESS))) ++ return -EINVAL; + + dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); + if (unlikely(!dev)) + return -EINVAL; + +- skb2 = skb_clone(skb, GFP_ATOMIC); +- if (unlikely(!skb2)) ++ clone = skb_clone(skb, GFP_ATOMIC); ++ if (unlikely(!clone)) + return -ENOMEM; + +- if (BPF_IS_REDIRECT_INGRESS(flags)) +- return dev_forward_skb(dev, skb2); ++ /* For direct write, we need to keep the invariant that the skbs ++ * we're dealing with need to be uncloned. Should uncloning fail ++ * here, we need to free the just generated clone to unclone once ++ * again. ++ */ ++ ret = bpf_try_make_head_writable(skb); ++ if (unlikely(ret)) { ++ kfree_skb(clone); ++ return -ENOMEM; ++ } + +- skb2->dev = dev; +- skb_sender_cpu_clear(skb2); +- return dev_queue_xmit(skb2); ++ return __bpf_redirect(clone, dev, flags); + } + +-const struct bpf_func_proto bpf_clone_redirect_proto = { ++static const struct bpf_func_proto bpf_clone_redirect_proto = { + .func = bpf_clone_redirect, + .gpl_only = false, + .ret_type = RET_INTEGER, +@@ -1432,42 +1977,38 @@ const struct bpf_func_proto bpf_clone_re + .arg3_type = ARG_ANYTHING, + }; + +-struct redirect_info { +- u32 ifindex; +- u32 flags; +-}; ++DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); ++EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); + +-static DEFINE_PER_CPU(struct redirect_info, redirect_info); +-static u64 bpf_redirect(u64 ifindex, u64 flags, u64 r3, u64 r4, u64 r5) ++BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) + { +- struct redirect_info *ri = this_cpu_ptr(&redirect_info); ++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); ++ ++ if (unlikely(flags & ~(BPF_F_INGRESS))) ++ return TC_ACT_SHOT; + +- ri->ifindex = ifindex; + ri->flags = flags; ++ ri->tgt_index = ifindex; ++ + return TC_ACT_REDIRECT; + } + + int skb_do_redirect(struct sk_buff *skb) + { +- struct redirect_info *ri = this_cpu_ptr(&redirect_info); ++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); + struct net_device *dev; + +- dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex); +- ri->ifindex = 0; ++ dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index); ++ ri->tgt_index = 0; + if (unlikely(!dev)) { + kfree_skb(skb); + return -EINVAL; + } + +- if (BPF_IS_REDIRECT_INGRESS(ri->flags)) +- return dev_forward_skb(dev, skb); +- +- skb->dev = dev; +- skb_sender_cpu_clear(skb); +- return dev_queue_xmit(skb); ++ return __bpf_redirect(skb, dev, ri->flags); + } + +-const struct bpf_func_proto bpf_redirect_proto = { ++static const struct bpf_func_proto bpf_redirect_proto = { + .func = bpf_redirect, + .gpl_only = false, + .ret_type = RET_INTEGER, +@@ -1475,50 +2016,75 @@ const struct bpf_func_proto bpf_redirect + .arg2_type = ARG_ANYTHING, + }; + +-static u64 bpf_get_cgroup_classid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb) + { +- return task_get_classid((struct sk_buff *) (unsigned long) r1); ++ /* If skb_clear_hash() was called due to mangling, we can ++ * trigger SW recalculation here. Later access to hash ++ * can then use the inline skb->hash via context directly ++ * instead of calling this helper again. ++ */ ++ return skb_get_hash(skb); + } + +-static const struct bpf_func_proto bpf_get_cgroup_classid_proto = { +- .func = bpf_get_cgroup_classid, +- .gpl_only = false, +- .ret_type = RET_INTEGER, +- .arg1_type = ARG_PTR_TO_CTX, ++static const struct bpf_func_proto bpf_get_hash_recalc_proto = { ++ .func = bpf_get_hash_recalc, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, + }; + +-static u64 bpf_get_route_realm(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb) + { +-#ifdef CONFIG_IP_ROUTE_CLASSID +- const struct dst_entry *dst; ++ /* After all direct packet write, this can be used once for ++ * triggering a lazy recalc on next skb_get_hash() invocation. ++ */ ++ skb_clear_hash(skb); ++ return 0; ++} + +- dst = skb_dst((struct sk_buff *) (unsigned long) r1); +- if (dst) +- return dst->tclassid; +-#endif ++static const struct bpf_func_proto bpf_set_hash_invalid_proto = { ++ .func = bpf_set_hash_invalid, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++}; ++ ++BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash) ++{ ++ /* Set user specified hash as L4(+), so that it gets returned ++ * on skb_get_hash() call unless BPF prog later on triggers a ++ * skb_clear_hash(). ++ */ ++ __skb_set_sw_hash(skb, hash, true); + return 0; + } + +-static const struct bpf_func_proto bpf_get_route_realm_proto = { +- .func = bpf_get_route_realm, +- .gpl_only = false, +- .ret_type = RET_INTEGER, +- .arg1_type = ARG_PTR_TO_CTX, ++static const struct bpf_func_proto bpf_set_hash_proto = { ++ .func = bpf_set_hash, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, + }; + +-static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5) ++BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto, ++ u16, vlan_tci) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- __be16 vlan_proto = (__force __be16) r2; ++ int ret; + + if (unlikely(vlan_proto != htons(ETH_P_8021Q) && + vlan_proto != htons(ETH_P_8021AD))) + vlan_proto = htons(ETH_P_8021Q); + +- return skb_vlan_push(skb, vlan_proto, vlan_tci); ++ bpf_push_mac_rcsum(skb); ++ ret = skb_vlan_push(skb, vlan_proto, vlan_tci); ++ bpf_pull_mac_rcsum(skb); ++ ++ bpf_compute_data_pointers(skb); ++ return ret; + } + +-const struct bpf_func_proto bpf_skb_vlan_push_proto = { ++static const struct bpf_func_proto bpf_skb_vlan_push_proto = { + .func = bpf_skb_vlan_push, + .gpl_only = false, + .ret_type = RET_INTEGER, +@@ -1526,116 +2092,401 @@ const struct bpf_func_proto bpf_skb_vlan + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, + }; +-EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto); + +-static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) ++BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; ++ int ret; ++ ++ bpf_push_mac_rcsum(skb); ++ ret = skb_vlan_pop(skb); ++ bpf_pull_mac_rcsum(skb); + +- return skb_vlan_pop(skb); ++ bpf_compute_data_pointers(skb); ++ return ret; + } + +-const struct bpf_func_proto bpf_skb_vlan_pop_proto = { ++static const struct bpf_func_proto bpf_skb_vlan_pop_proto = { + .func = bpf_skb_vlan_pop, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + }; +-EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto); + +-bool bpf_helper_changes_skb_data(void *func) ++BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type) + { +- if (func == bpf_skb_vlan_push) +- return true; +- if (func == bpf_skb_vlan_pop) +- return true; +- if (func == bpf_skb_store_bytes) +- return true; +- if (func == bpf_l3_csum_replace) +- return true; +- if (func == bpf_l4_csum_replace) +- return true; ++ /* We only allow a restricted subset to be changed for now. */ ++ if (unlikely(!skb_pkt_type_ok(skb->pkt_type) || ++ !skb_pkt_type_ok(pkt_type))) ++ return -EINVAL; + +- return false; ++ skb->pkt_type = pkt_type; ++ return 0; ++} ++ ++static const struct bpf_func_proto bpf_skb_change_type_proto = { ++ .func = bpf_skb_change_type, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++}; ++ ++#define BPF_F_ADJ_ROOM_ENCAP_L3_MASK (BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \ ++ BPF_F_ADJ_ROOM_ENCAP_L3_IPV6) ++ ++#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \ ++ BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \ ++ BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \ ++ BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \ ++ BPF_F_ADJ_ROOM_ENCAP_L2( \ ++ BPF_ADJ_ROOM_ENCAP_L2_MASK)) ++ ++#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC ++ ++static u32 __bpf_skb_min_len(const struct sk_buff *skb) ++{ ++ u32 min_len = skb_network_offset(skb); ++ ++ if (skb_transport_header_was_set(skb)) ++ min_len = skb_transport_offset(skb); ++ if (skb->ip_summed == CHECKSUM_PARTIAL) ++ min_len = skb_checksum_start_offset(skb) + ++ skb->csum_offset + sizeof(__sum16); ++ return min_len; + } + +-static u64 bpf_skb_get_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) ++static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- struct bpf_tunnel_key *to = (struct bpf_tunnel_key *) (long) r2; +- struct ip_tunnel_info *info = skb_tunnel_info(skb); ++ unsigned int old_len = skb->len; ++ int ret; + +- if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags || !info)) ++ ret = __skb_grow_rcsum(skb, new_len); ++ if (!ret) ++ memset(skb->data + old_len, 0, new_len - old_len); ++ return ret; ++} ++ ++static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) ++{ ++ return __skb_trim_rcsum(skb, new_len); ++} ++ ++static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len, ++ u64 flags) ++{ ++ u32 max_len = BPF_SKB_MAX_LEN; ++ u32 min_len = __bpf_skb_min_len(skb); ++ int ret; ++ ++ if (unlikely(flags || new_len > max_len || new_len < min_len)) + return -EINVAL; +- if (ip_tunnel_info_af(info) != AF_INET) ++ if (skb->encapsulation) ++ return -ENOTSUPP; ++ ++ /* The basic idea of this helper is that it's performing the ++ * needed work to either grow or trim an skb, and eBPF program ++ * rewrites the rest via helpers like bpf_skb_store_bytes(), ++ * bpf_lX_csum_replace() and others rather than passing a raw ++ * buffer here. This one is a slow path helper and intended ++ * for replies with control messages. ++ * ++ * Like in bpf_skb_change_proto(), we want to keep this rather ++ * minimal and without protocol specifics so that we are able ++ * to separate concerns as in bpf_skb_store_bytes() should only ++ * be the one responsible for writing buffers. ++ * ++ * It's really expected to be a slow path operation here for ++ * control message replies, so we're implicitly linearizing, ++ * uncloning and drop offloads from the skb by this. ++ */ ++ ret = __bpf_try_make_writable(skb, skb->len); ++ if (!ret) { ++ if (new_len > skb->len) ++ ret = bpf_skb_grow_rcsum(skb, new_len); ++ else if (new_len < skb->len) ++ ret = bpf_skb_trim_rcsum(skb, new_len); ++ if (!ret && skb_is_gso(skb)) ++ skb_gso_reset(skb); ++ } ++ return ret; ++} ++ ++BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len, ++ u64, flags) ++{ ++ int ret = __bpf_skb_change_tail(skb, new_len, flags); ++ ++ bpf_compute_data_pointers(skb); ++ return ret; ++} ++ ++static const struct bpf_func_proto bpf_skb_change_tail_proto = { ++ .func = bpf_skb_change_tail, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_ANYTHING, ++}; ++ ++static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room, ++ u64 flags) ++{ ++ u32 max_len = BPF_SKB_MAX_LEN; ++ u32 new_len = skb->len + head_room; ++ int ret; ++ ++ if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) || ++ new_len < skb->len)) + return -EINVAL; + +- to->tunnel_id = be64_to_cpu(info->key.tun_id); +- to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src); ++ ret = skb_cow(skb, head_room); ++ if (likely(!ret)) { ++ /* Idea for this helper is that we currently only ++ * allow to expand on mac header. This means that ++ * skb->protocol network header, etc, stay as is. ++ * Compared to bpf_skb_change_tail(), we're more ++ * flexible due to not needing to linearize or ++ * reset GSO. Intention for this helper is to be ++ * used by an L3 skb that needs to push mac header ++ * for redirection into L2 device. ++ */ ++ __skb_push(skb, head_room); ++ memset(skb->data, 0, head_room); ++ skb_reset_mac_header(skb); ++ skb_reset_mac_len(skb); ++ } + +- return 0; ++ return ret; + } + +-const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = { +- .func = bpf_skb_get_tunnel_key, ++BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room, ++ u64, flags) ++{ ++ int ret = __bpf_skb_change_head(skb, head_room, flags); ++ ++ bpf_compute_data_pointers(skb); ++ return ret; ++} ++ ++static const struct bpf_func_proto bpf_skb_change_head_proto = { ++ .func = bpf_skb_change_head, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, +- .arg2_type = ARG_PTR_TO_STACK, +- .arg3_type = ARG_CONST_STACK_SIZE, +- .arg4_type = ARG_ANYTHING, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_ANYTHING, + }; + +-static struct metadata_dst __percpu *md_dst; ++void bpf_clear_redirect_map(struct bpf_map *map) ++{ ++ struct bpf_redirect_info *ri; ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ ri = per_cpu_ptr(&bpf_redirect_info, cpu); ++ /* Avoid polluting remote cacheline due to writes if ++ * not needed. Once we pass this test, we need the ++ * cmpxchg() to make sure it hasn't been changed in ++ * the meantime by remote CPU. ++ */ ++ if (unlikely(READ_ONCE(ri->map) == map)) ++ cmpxchg(&ri->map, map, NULL); ++ } ++} ++ ++static unsigned long bpf_skb_copy(void *dst_buff, const void *skb, ++ unsigned long off, unsigned long len) ++{ ++ void *ptr = skb_header_pointer(skb, off, len, dst_buff); ++ ++ if (unlikely(!ptr)) ++ return len; ++ if (ptr != dst_buff) ++ memcpy(dst_buff, ptr, len); ++ ++ return 0; ++} + +-static u64 bpf_skb_set_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) ++BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map, ++ u64, flags, void *, meta, u64, meta_size) + { +- struct sk_buff *skb = (struct sk_buff *) (long) r1; +- struct bpf_tunnel_key *from = (struct bpf_tunnel_key *) (long) r2; +- struct metadata_dst *md = this_cpu_ptr(md_dst); +- struct ip_tunnel_info *info; ++ u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32; + +- if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags)) ++ if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK))) + return -EINVAL; ++ if (unlikely(skb_size > skb->len)) ++ return -EFAULT; + +- skb_dst_drop(skb); +- dst_hold((struct dst_entry *) md); +- skb_dst_set(skb, (struct dst_entry *) md); ++ return bpf_event_output(map, flags, meta, meta_size, skb, skb_size, ++ bpf_skb_copy); ++} + +- info = &md->u.tun_info; +- info->mode = IP_TUNNEL_INFO_TX; +- info->key.tun_flags = TUNNEL_KEY; +- info->key.tun_id = cpu_to_be64(from->tunnel_id); +- info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4); ++static const struct bpf_func_proto bpf_skb_event_output_proto = { ++ .func = bpf_skb_event_output, ++ .gpl_only = true, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_CONST_MAP_PTR, ++ .arg3_type = ARG_ANYTHING, ++ .arg4_type = ARG_PTR_TO_MEM, ++ .arg5_type = ARG_CONST_SIZE_OR_ZERO, ++}; ++ ++ ++const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; ++EXPORT_SYMBOL_GPL(ipv6_bpf_stub); ++ ++#ifdef CONFIG_XFRM ++BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index, ++ struct bpf_xfrm_state *, to, u32, size, u64, flags) ++{ ++ const struct sec_path *sp = skb_sec_path(skb); ++ const struct xfrm_state *x; ++ ++ if (!sp || unlikely(index >= sp->len || flags)) ++ goto err_clear; ++ ++ x = sp->xvec[index]; ++ ++ if (unlikely(size != sizeof(struct bpf_xfrm_state))) ++ goto err_clear; ++ ++ to->reqid = x->props.reqid; ++ to->spi = x->id.spi; ++ to->family = x->props.family; ++ to->ext = 0; ++ ++ if (to->family == AF_INET6) { ++ memcpy(to->remote_ipv6, x->props.saddr.a6, ++ sizeof(to->remote_ipv6)); ++ } else { ++ to->remote_ipv4 = x->props.saddr.a4; ++ memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3); ++ } + + return 0; ++err_clear: ++ memset(to, 0, size); ++ return -EINVAL; + } + +-const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { +- .func = bpf_skb_set_tunnel_key, ++static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { ++ .func = bpf_skb_get_xfrm_state, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, +- .arg2_type = ARG_PTR_TO_STACK, +- .arg3_type = ARG_CONST_STACK_SIZE, +- .arg4_type = ARG_ANYTHING, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, ++ .arg4_type = ARG_CONST_SIZE, ++ .arg5_type = ARG_ANYTHING, + }; ++#endif ++ + +-static const struct bpf_func_proto *bpf_get_skb_set_tunnel_key_proto(void) ++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) ++static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, ++ bool ingress) + { +- if (!md_dst) { +- /* race is not possible, since it's called from +- * verifier that is holding verifier mutex +- */ +- md_dst = metadata_dst_alloc_percpu(0, GFP_KERNEL); +- if (!md_dst) +- return NULL; ++ return bpf_lwt_push_ip_encap(skb, hdr, len, ingress); ++} ++#endif ++ ++BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr, ++ u32, len) ++{ ++ switch (type) { ++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) ++ case BPF_LWT_ENCAP_IP: ++ return bpf_push_ip_encap(skb, hdr, len, true /* ingress */); ++#endif ++ default: ++ return -EINVAL; + } +- return &bpf_skb_set_tunnel_key_proto; ++} ++ ++BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type, ++ void *, hdr, u32, len) ++{ ++ switch (type) { ++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) ++ case BPF_LWT_ENCAP_IP: ++ return bpf_push_ip_encap(skb, hdr, len, false /* egress */); ++#endif ++ default: ++ return -EINVAL; ++ } ++} ++ ++static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { ++ .func = bpf_lwt_in_push_encap, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_PTR_TO_MEM, ++ .arg4_type = ARG_CONST_SIZE ++}; ++ ++static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { ++ .func = bpf_lwt_xmit_push_encap, ++ .gpl_only = false, ++ .ret_type = RET_INTEGER, ++ .arg1_type = ARG_PTR_TO_CTX, ++ .arg2_type = ARG_ANYTHING, ++ .arg3_type = ARG_PTR_TO_MEM, ++ .arg4_type = ARG_CONST_SIZE ++}; ++ ++bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++BPF_CALL_1(bpf_tcp_sock, struct sock *, sk) ++{ ++ if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP) ++ return (unsigned long)sk; ++ ++ return (unsigned long)NULL; ++} ++ ++const struct bpf_func_proto bpf_tcp_sock_proto = { ++ .func = bpf_tcp_sock, ++ .gpl_only = false, ++ .ret_type = RET_PTR_TO_TCP_SOCK_OR_NULL, ++ .arg1_type = ARG_PTR_TO_SOCK_COMMON, ++}; ++ ++bool bpf_helper_changes_pkt_data(void *func) ++{ ++ if (func == bpf_skb_vlan_push || ++ func == bpf_skb_vlan_pop || ++ func == bpf_skb_store_bytes || ++ func == bpf_skb_change_head || ++ func == bpf_skb_change_tail || ++ func == bpf_skb_pull_data || ++ func == bpf_clone_redirect || ++ func == bpf_l3_csum_replace || ++ func == bpf_l4_csum_replace || ++ func == bpf_lwt_in_push_encap || ++ func == bpf_lwt_xmit_push_encap) ++ return true; ++ ++ return false; + } + + static const struct bpf_func_proto * +-sk_filter_func_proto(enum bpf_func_id func_id) ++bpf_base_func_proto(enum bpf_func_id func_id) + { + switch (func_id) { + case BPF_FUNC_map_lookup_elem: +@@ -1644,283 +2495,1168 @@ sk_filter_func_proto(enum bpf_func_id fu + return &bpf_map_update_elem_proto; + case BPF_FUNC_map_delete_elem: + return &bpf_map_delete_elem_proto; ++ case BPF_FUNC_map_push_elem: ++ return &bpf_map_push_elem_proto; ++ case BPF_FUNC_map_pop_elem: ++ return &bpf_map_pop_elem_proto; ++ case BPF_FUNC_map_peek_elem: ++ return &bpf_map_peek_elem_proto; + case BPF_FUNC_get_prandom_u32: + return &bpf_get_prandom_u32_proto; + case BPF_FUNC_get_smp_processor_id: +- return &bpf_get_smp_processor_id_proto; ++ return &bpf_get_raw_smp_processor_id_proto; ++ case BPF_FUNC_get_numa_node_id: ++ return &bpf_get_numa_node_id_proto; + case BPF_FUNC_tail_call: + return &bpf_tail_call_proto; + case BPF_FUNC_ktime_get_ns: + return &bpf_ktime_get_ns_proto; ++ default: ++ break; ++ } ++ ++ if (!capable(CAP_SYS_ADMIN)) ++ return NULL; ++ ++ switch (func_id) { ++ case BPF_FUNC_spin_lock: ++ return &bpf_spin_lock_proto; ++ case BPF_FUNC_spin_unlock: ++ return &bpf_spin_unlock_proto; + case BPF_FUNC_trace_printk: +- if (capable(CAP_SYS_ADMIN)) +- return bpf_get_trace_printk_proto(); ++ return bpf_get_trace_printk_proto(); + default: + return NULL; + } + } + + static const struct bpf_func_proto * +-tc_cls_act_func_proto(enum bpf_func_id func_id) ++sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ return bpf_base_func_proto(func_id); ++} ++ ++static const struct bpf_func_proto * ++sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ return bpf_base_func_proto(func_id); ++} ++ ++const struct bpf_func_proto bpf_sk_storage_get_proto __weak; ++const struct bpf_func_proto bpf_sk_storage_delete_proto __weak; ++ ++static const struct bpf_func_proto * ++tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) + { + switch (func_id) { + case BPF_FUNC_skb_store_bytes: + return &bpf_skb_store_bytes_proto; ++ case BPF_FUNC_skb_load_bytes: ++ return &bpf_skb_load_bytes_proto; ++ case BPF_FUNC_skb_load_bytes_relative: ++ return &bpf_skb_load_bytes_relative_proto; ++ case BPF_FUNC_skb_pull_data: ++ return &bpf_skb_pull_data_proto; ++ case BPF_FUNC_csum_diff: ++ return &bpf_csum_diff_proto; ++ case BPF_FUNC_csum_update: ++ return &bpf_csum_update_proto; + case BPF_FUNC_l3_csum_replace: + return &bpf_l3_csum_replace_proto; + case BPF_FUNC_l4_csum_replace: + return &bpf_l4_csum_replace_proto; + case BPF_FUNC_clone_redirect: + return &bpf_clone_redirect_proto; +- case BPF_FUNC_get_cgroup_classid: +- return &bpf_get_cgroup_classid_proto; + case BPF_FUNC_skb_vlan_push: + return &bpf_skb_vlan_push_proto; + case BPF_FUNC_skb_vlan_pop: + return &bpf_skb_vlan_pop_proto; +- case BPF_FUNC_skb_get_tunnel_key: +- return &bpf_skb_get_tunnel_key_proto; +- case BPF_FUNC_skb_set_tunnel_key: +- return bpf_get_skb_set_tunnel_key_proto(); ++ case BPF_FUNC_skb_change_type: ++ return &bpf_skb_change_type_proto; ++ case BPF_FUNC_skb_change_tail: ++ return &bpf_skb_change_tail_proto; ++ case BPF_FUNC_redirect: ++ return &bpf_redirect_proto; ++ case BPF_FUNC_get_hash_recalc: ++ return &bpf_get_hash_recalc_proto; ++ case BPF_FUNC_set_hash_invalid: ++ return &bpf_set_hash_invalid_proto; ++ case BPF_FUNC_set_hash: ++ return &bpf_set_hash_proto; ++ case BPF_FUNC_perf_event_output: ++ return &bpf_skb_event_output_proto; ++ case BPF_FUNC_get_smp_processor_id: ++ return &bpf_get_smp_processor_id_proto; ++#ifdef CONFIG_XFRM ++ case BPF_FUNC_skb_get_xfrm_state: ++ return &bpf_skb_get_xfrm_state_proto; ++#endif ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++static const struct bpf_func_proto * ++xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++const struct bpf_func_proto bpf_sock_map_update_proto __weak; ++const struct bpf_func_proto bpf_sock_hash_update_proto __weak; ++ ++static const struct bpf_func_proto * ++sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++const struct bpf_func_proto bpf_msg_redirect_map_proto __weak; ++const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak; ++ ++static const struct bpf_func_proto * ++sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++const struct bpf_func_proto bpf_sk_redirect_map_proto __weak; ++const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak; ++ ++static const struct bpf_func_proto * ++sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ return bpf_base_func_proto(func_id); ++} ++ ++static const struct bpf_func_proto * ++flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++static const struct bpf_func_proto * ++lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ case BPF_FUNC_skb_load_bytes: ++ return &bpf_skb_load_bytes_proto; ++ case BPF_FUNC_skb_pull_data: ++ return &bpf_skb_pull_data_proto; ++ case BPF_FUNC_csum_diff: ++ return &bpf_csum_diff_proto; ++ case BPF_FUNC_get_hash_recalc: ++ return &bpf_get_hash_recalc_proto; ++ case BPF_FUNC_perf_event_output: ++ return &bpf_skb_event_output_proto; ++ case BPF_FUNC_get_smp_processor_id: ++ return &bpf_get_smp_processor_id_proto; ++ default: ++ return bpf_base_func_proto(func_id); ++ } ++} ++ ++static const struct bpf_func_proto * ++lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { ++ case BPF_FUNC_lwt_push_encap: ++ return &bpf_lwt_in_push_encap_proto; ++ default: ++ return lwt_out_func_proto(func_id, prog); ++ } ++} ++ ++static const struct bpf_func_proto * ++lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) ++{ ++ switch (func_id) { + case BPF_FUNC_redirect: + return &bpf_redirect_proto; +- case BPF_FUNC_get_route_realm: +- return &bpf_get_route_realm_proto; ++ case BPF_FUNC_clone_redirect: ++ return &bpf_clone_redirect_proto; ++ case BPF_FUNC_skb_change_tail: ++ return &bpf_skb_change_tail_proto; ++ case BPF_FUNC_skb_change_head: ++ return &bpf_skb_change_head_proto; ++ case BPF_FUNC_skb_store_bytes: ++ return &bpf_skb_store_bytes_proto; ++ case BPF_FUNC_csum_update: ++ return &bpf_csum_update_proto; ++ case BPF_FUNC_l3_csum_replace: ++ return &bpf_l3_csum_replace_proto; ++ case BPF_FUNC_l4_csum_replace: ++ return &bpf_l4_csum_replace_proto; ++ case BPF_FUNC_set_hash_invalid: ++ return &bpf_set_hash_invalid_proto; ++ case BPF_FUNC_lwt_push_encap: ++ return &bpf_lwt_xmit_push_encap_proto; + default: +- return sk_filter_func_proto(func_id); ++ return lwt_out_func_proto(func_id, prog); + } + } + +-static bool __is_valid_access(int off, int size, enum bpf_access_type type) ++static const struct bpf_func_proto * ++lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) + { +- /* check bounds */ ++ switch (func_id) { ++ default: ++ return lwt_out_func_proto(func_id, prog); ++ } ++} ++ ++static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ const int size_default = sizeof(__u32); ++ + if (off < 0 || off >= sizeof(struct __sk_buff)) + return false; + +- /* disallow misaligned access */ ++ /* The verifier guarantees that size > 0. */ + if (off % size != 0) + return false; + +- /* all __sk_buff fields are __u32 */ +- if (size != 4) ++ switch (off) { ++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): ++ if (off + size > offsetofend(struct __sk_buff, cb[4])) ++ return false; ++ break; ++ case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]): ++ case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]): ++ case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4): ++ case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4): ++ case bpf_ctx_range(struct __sk_buff, data): ++ case bpf_ctx_range(struct __sk_buff, data_meta): ++ case bpf_ctx_range(struct __sk_buff, data_end): ++ if (size != size_default) ++ return false; ++ break; ++ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys): + return false; ++ case bpf_ctx_range(struct __sk_buff, tstamp): ++ if (size != sizeof(__u64)) ++ return false; ++ break; ++ case offsetof(struct __sk_buff, sk): ++ if (type == BPF_WRITE || size != sizeof(__u64)) ++ return false; ++ info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL; ++ break; ++ default: ++ /* Only narrow read access allowed for now. */ ++ if (type == BPF_WRITE) { ++ if (size != size_default) ++ return false; ++ } else { ++ bpf_ctx_record_field_size(info, size_default); ++ if (!bpf_ctx_narrow_access_ok(off, size, size_default)) ++ return false; ++ } ++ } + + return true; + } + + static bool sk_filter_is_valid_access(int off, int size, +- enum bpf_access_type type) ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) + { +- if (off == offsetof(struct __sk_buff, tc_classid)) ++ return false; ++} ++ ++static bool lwt_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ switch (off) { ++ case bpf_ctx_range(struct __sk_buff, tc_classid): ++ case bpf_ctx_range_till(struct __sk_buff, family, local_port): ++ case bpf_ctx_range(struct __sk_buff, data_meta): ++ case bpf_ctx_range(struct __sk_buff, tstamp): ++ case bpf_ctx_range(struct __sk_buff, wire_len): + return false; ++ } + + if (type == BPF_WRITE) { + switch (off) { +- case offsetof(struct __sk_buff, cb[0]) ... +- offsetof(struct __sk_buff, cb[4]): ++ case bpf_ctx_range(struct __sk_buff, mark): ++ case bpf_ctx_range(struct __sk_buff, priority): ++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): + break; + default: + return false; + } + } + +- return __is_valid_access(off, size, type); ++ switch (off) { ++ case bpf_ctx_range(struct __sk_buff, data): ++ info->reg_type = PTR_TO_PACKET; ++ break; ++ case bpf_ctx_range(struct __sk_buff, data_end): ++ info->reg_type = PTR_TO_PACKET_END; ++ break; ++ } ++ ++ return bpf_skb_is_valid_access(off, size, type, prog, info); + } + +-static bool tc_cls_act_is_valid_access(int off, int size, +- enum bpf_access_type type) ++ ++bool bpf_sock_common_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ switch (off) { ++ case bpf_ctx_range_till(struct bpf_sock, type, priority): ++ return false; ++ default: ++ return bpf_sock_is_valid_access(off, size, type, info); ++ } ++} ++ ++bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static bool sock_filter_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write, ++ const struct bpf_prog *prog) ++{ ++ /* Neither direct read nor direct write requires any preliminary ++ * action. ++ */ ++ return 0; ++} ++ ++static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write, ++ const struct bpf_prog *prog, int drop_verdict) ++{ ++ struct bpf_insn *insn = insn_buf; ++ ++ if (!direct_write) ++ return 0; ++ ++ /* if (!skb->cloned) ++ * goto start; ++ * ++ * (Fast-path, otherwise approximation that we might be ++ * a clone, do the rest in helper.) ++ */ ++ *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET()); ++ *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7); ++ ++ /* ret = bpf_skb_pull_data(skb, 0); */ ++ *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); ++ *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2); ++ *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, ++ BPF_FUNC_skb_pull_data); ++ /* if (!ret) ++ * goto restore; ++ * return TC_ACT_SHOT; ++ */ ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2); ++ *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict); ++ *insn++ = BPF_EXIT_INSN(); ++ ++ /* restore: */ ++ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6); ++ /* start: */ ++ *insn++ = prog->insnsi[0]; ++ ++ return insn - insn_buf; ++} ++ ++static int bpf_gen_ld_abs(const struct bpf_insn *orig, ++ struct bpf_insn *insn_buf) ++{ ++ bool indirect = BPF_MODE(orig->code) == BPF_IND; ++ struct bpf_insn *insn = insn_buf; ++ ++ if (!indirect) { ++ *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm); ++ } else { ++ *insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg); ++ if (orig->imm) ++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm); ++ } ++ /* We're guaranteed here that CTX is in R6. */ ++ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX); ++ ++ switch (BPF_SIZE(orig->code)) { ++ case BPF_B: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache); ++ break; ++ case BPF_H: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache); ++ break; ++ case BPF_W: ++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache); ++ break; ++ } ++ ++ *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2); ++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); ++ *insn++ = BPF_EXIT_INSN(); ++ ++ return insn - insn_buf; ++} ++ ++static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write, ++ const struct bpf_prog *prog) + { +- if (off == offsetof(struct __sk_buff, tc_classid)) +- return type == BPF_WRITE ? true : false; ++ return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT); ++} + ++static bool tc_cls_act_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ + if (type == BPF_WRITE) { + switch (off) { +- case offsetof(struct __sk_buff, mark): +- case offsetof(struct __sk_buff, tc_index): +- case offsetof(struct __sk_buff, priority): +- case offsetof(struct __sk_buff, cb[0]) ... +- offsetof(struct __sk_buff, cb[4]): ++ case bpf_ctx_range(struct __sk_buff, mark): ++ case bpf_ctx_range(struct __sk_buff, tc_index): ++ case bpf_ctx_range(struct __sk_buff, priority): ++ case bpf_ctx_range(struct __sk_buff, tc_classid): ++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): ++ case bpf_ctx_range(struct __sk_buff, tstamp): ++ case bpf_ctx_range(struct __sk_buff, queue_mapping): + break; + default: + return false; + } + } +- return __is_valid_access(off, size, type); ++ ++ switch (off) { ++ case bpf_ctx_range(struct __sk_buff, data): ++ info->reg_type = PTR_TO_PACKET; ++ break; ++ case bpf_ctx_range(struct __sk_buff, data_meta): ++ info->reg_type = PTR_TO_PACKET_META; ++ break; ++ case bpf_ctx_range(struct __sk_buff, data_end): ++ info->reg_type = PTR_TO_PACKET_END; ++ break; ++ case bpf_ctx_range_till(struct __sk_buff, family, local_port): ++ return false; ++ } ++ ++ return bpf_skb_is_valid_access(off, size, type, prog, info); ++} ++ ++static bool xdp_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++void bpf_warn_invalid_xdp_action(u32 act) ++{ ++} ++EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action); ++ ++static bool sock_addr_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static bool sock_ops_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write, ++ const struct bpf_prog *prog) ++{ ++ return bpf_unclone_prologue(insn_buf, direct_write, prog, SK_DROP); ++} ++ ++static bool sk_skb_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static bool sk_msg_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; ++} ++ ++static bool flow_dissector_is_valid_access(int off, int size, ++ enum bpf_access_type type, ++ const struct bpf_prog *prog, ++ struct bpf_insn_access_aux *info) ++{ ++ return false; + } + +-static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg, +- int src_reg, int ctx_off, +- struct bpf_insn *insn_buf, +- struct bpf_prog *prog) ++static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++ ++{ ++ return 0; ++} ++ ++static u32 bpf_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) + { + struct bpf_insn *insn = insn_buf; ++ int off; + +- switch (ctx_off) { ++ switch (si->off) { + case offsetof(struct __sk_buff, len): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); +- +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, len)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, len, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, protocol): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); +- +- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, +- offsetof(struct sk_buff, protocol)); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, protocol, 2, ++ target_size)); + break; + + case offsetof(struct __sk_buff, vlan_proto): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2); +- +- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, +- offsetof(struct sk_buff, vlan_proto)); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, vlan_proto, 2, ++ target_size)); + break; + + case offsetof(struct __sk_buff, priority): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, priority) != 4); +- + if (type == BPF_WRITE) +- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, priority)); ++ *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, priority, 4, ++ target_size)); + else +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, priority)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, priority, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, ingress_ifindex): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, skb_iif) != 4); +- +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, skb_iif)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, skb_iif, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, ifindex): +- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); +- +- *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), +- dst_reg, src_reg, ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), ++ si->dst_reg, si->src_reg, + offsetof(struct sk_buff, dev)); +- *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1); +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, dst_reg, +- offsetof(struct net_device, ifindex)); ++ *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct net_device, ifindex, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, hash): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); +- +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, hash)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, hash, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, mark): +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); +- + if (type == BPF_WRITE) +- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, mark)); ++ *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, mark, 4, ++ target_size)); + else +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, +- offsetof(struct sk_buff, mark)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, mark, 4, ++ target_size)); + break; + + case offsetof(struct __sk_buff, pkt_type): +- return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn); ++ *target_size = 1; ++ *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, ++ PKT_TYPE_OFFSET()); ++ *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX); ++#ifdef __BIG_ENDIAN_BITFIELD ++ *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5); ++#endif ++ break; + + case offsetof(struct __sk_buff, queue_mapping): +- return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn); ++ if (type == BPF_WRITE) { ++ *insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, USHRT_MAX, 1); ++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, ++ queue_mapping, ++ 2, target_size)); ++ } else { ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, ++ queue_mapping, ++ 2, target_size)); ++ } ++ break; + + case offsetof(struct __sk_buff, vlan_present): +- return convert_skb_access(SKF_AD_VLAN_TAG_PRESENT, +- dst_reg, src_reg, insn); ++ *target_size = 1; ++ *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, ++ PKT_VLAN_PRESENT_OFFSET()); ++ if (PKT_VLAN_PRESENT_BIT) ++ *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT); ++ if (PKT_VLAN_PRESENT_BIT < 7) ++ *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, 1); ++ break; + + case offsetof(struct __sk_buff, vlan_tci): +- return convert_skb_access(SKF_AD_VLAN_TAG, +- dst_reg, src_reg, insn); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, vlan_tci, 2, ++ target_size)); ++ break; + + case offsetof(struct __sk_buff, cb[0]) ... +- offsetof(struct __sk_buff, cb[4]): ++ offsetofend(struct __sk_buff, cb[4]) - 1: + BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, data) < 20); ++ BUILD_BUG_ON((offsetof(struct sk_buff, cb) + ++ offsetof(struct qdisc_skb_cb, data)) % ++ sizeof(__u64)); + + prog->cb_access = 1; +- ctx_off -= offsetof(struct __sk_buff, cb[0]); +- ctx_off += offsetof(struct sk_buff, cb); +- ctx_off += offsetof(struct qdisc_skb_cb, data); ++ off = si->off; ++ off -= offsetof(struct __sk_buff, cb[0]); ++ off += offsetof(struct sk_buff, cb); ++ off += offsetof(struct qdisc_skb_cb, data); + if (type == BPF_WRITE) +- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, ctx_off); ++ *insn++ = BPF_STX_MEM(BPF_SIZE(si->code), si->dst_reg, ++ si->src_reg, off); + else +- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, ctx_off); ++ *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg, ++ si->src_reg, off); + break; + + case offsetof(struct __sk_buff, tc_classid): +- ctx_off -= offsetof(struct __sk_buff, tc_classid); +- ctx_off += offsetof(struct sk_buff, cb); +- ctx_off += offsetof(struct qdisc_skb_cb, tc_classid); +- WARN_ON(type != BPF_WRITE); +- *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, ctx_off); ++ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, tc_classid) != 2); ++ ++ off = si->off; ++ off -= offsetof(struct __sk_buff, tc_classid); ++ off += offsetof(struct sk_buff, cb); ++ off += offsetof(struct qdisc_skb_cb, tc_classid); ++ *target_size = 2; ++ if (type == BPF_WRITE) ++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, ++ si->src_reg, off); ++ else ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, ++ si->src_reg, off); ++ break; ++ ++ case offsetof(struct __sk_buff, data): ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, data)); ++ break; ++ ++ case offsetof(struct __sk_buff, data_meta): ++ off = si->off; ++ off -= offsetof(struct __sk_buff, data_meta); ++ off += offsetof(struct sk_buff, cb); ++ off += offsetof(struct bpf_skb_data_end, data_meta); ++ *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, ++ si->src_reg, off); ++ break; ++ ++ case offsetof(struct __sk_buff, data_end): ++ off = si->off; ++ off -= offsetof(struct __sk_buff, data_end); ++ off += offsetof(struct sk_buff, cb); ++ off += offsetof(struct bpf_skb_data_end, data_end); ++ *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, ++ si->src_reg, off); + break; + + case offsetof(struct __sk_buff, tc_index): + #ifdef CONFIG_NET_SCHED +- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2); +- + if (type == BPF_WRITE) +- *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, +- offsetof(struct sk_buff, tc_index)); ++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, tc_index, 2, ++ target_size)); + else +- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, +- offsetof(struct sk_buff, tc_index)); +- break; ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, tc_index, 2, ++ target_size)); + #else ++ *target_size = 2; + if (type == BPF_WRITE) +- *insn++ = BPF_MOV64_REG(dst_reg, dst_reg); ++ *insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg); + else +- *insn++ = BPF_MOV64_IMM(dst_reg, 0); ++ *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); ++#endif ++ break; ++ ++ case offsetof(struct __sk_buff, napi_id): ++ *target_size = 4; ++ *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); ++ break; ++ case offsetof(struct __sk_buff, family): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_family) != 2); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct sock_common, ++ skc_family, ++ 2, target_size)); ++ break; ++ case offsetof(struct __sk_buff, remote_ip4): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_daddr) != 4); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct sock_common, ++ skc_daddr, ++ 4, target_size)); ++ break; ++ case offsetof(struct __sk_buff, local_ip4): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, ++ skc_rcv_saddr) != 4); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct sock_common, ++ skc_rcv_saddr, ++ 4, target_size)); ++ break; ++ case offsetof(struct __sk_buff, remote_ip6[0]) ... ++ offsetof(struct __sk_buff, remote_ip6[3]): ++#if IS_ENABLED(CONFIG_IPV6) ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, ++ skc_v6_daddr.s6_addr32[0]) != 4); ++ ++ off = si->off; ++ off -= offsetof(struct __sk_buff, remote_ip6[0]); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ offsetof(struct sock_common, ++ skc_v6_daddr.s6_addr32[0]) + ++ off); ++#else ++ *insn++ = BPF_MOV32_IMM(si->dst_reg, 0); ++#endif ++ break; ++ case offsetof(struct __sk_buff, local_ip6[0]) ... ++ offsetof(struct __sk_buff, local_ip6[3]): ++#if IS_ENABLED(CONFIG_IPV6) ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, ++ skc_v6_rcv_saddr.s6_addr32[0]) != 4); ++ ++ off = si->off; ++ off -= offsetof(struct __sk_buff, local_ip6[0]); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ offsetof(struct sock_common, ++ skc_v6_rcv_saddr.s6_addr32[0]) + ++ off); ++#else ++ *insn++ = BPF_MOV32_IMM(si->dst_reg, 0); ++#endif ++ break; ++ ++ case offsetof(struct __sk_buff, remote_port): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_dport) != 2); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct sock_common, ++ skc_dport, ++ 2, target_size)); ++#ifndef __BIG_ENDIAN_BITFIELD ++ *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16); ++#endif + break; ++ ++ case offsetof(struct __sk_buff, local_port): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_num) != 2); ++ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct sock_common, ++ skc_num, 2, target_size)); ++ break; ++ ++ case offsetof(struct __sk_buff, tstamp): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tstamp) != 8); ++ ++ if (type == BPF_WRITE) ++ *insn++ = BPF_STX_MEM(BPF_DW, ++ si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, ++ tstamp, 8, ++ target_size)); ++ else ++ *insn++ = BPF_LDX_MEM(BPF_DW, ++ si->dst_reg, si->src_reg, ++ bpf_target_off(struct sk_buff, ++ tstamp, 8, ++ target_size)); ++ break; ++ ++ case offsetof(struct __sk_buff, gso_segs): ++ /* si->dst_reg = skb_shinfo(SKB); */ ++#ifdef NET_SKBUFF_DATA_USES_OFFSET ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end), ++ BPF_REG_AX, si->src_reg, ++ offsetof(struct sk_buff, end)); ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, head)); ++ *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX); ++#else ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, end)); + #endif ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs), ++ si->dst_reg, si->dst_reg, ++ bpf_target_off(struct skb_shared_info, ++ gso_segs, 2, ++ target_size)); ++ break; ++ case offsetof(struct __sk_buff, wire_len): ++ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, pkt_len) != 4); ++ ++ off = si->off; ++ off -= offsetof(struct __sk_buff, wire_len); ++ off += offsetof(struct sk_buff, cb); ++ off += offsetof(struct qdisc_skb_cb, pkt_len); ++ *target_size = 4; ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off); ++ break; ++ ++ case offsetof(struct __sk_buff, sk): ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, sk)); ++ break; + } + + return insn - insn_buf; + } + +-static const struct bpf_verifier_ops sk_filter_ops = { +- .get_func_proto = sk_filter_func_proto, +- .is_valid_access = sk_filter_is_valid_access, +- .convert_ctx_access = bpf_net_convert_ctx_access, ++u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ struct bpf_insn *insn = insn_buf; ++ ++ switch (si->off) { ++ case offsetof(struct __sk_buff, ifindex): ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), ++ si->dst_reg, si->src_reg, ++ offsetof(struct sk_buff, dev)); ++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, ++ bpf_target_off(struct net_device, ifindex, 4, ++ target_size)); ++ break; ++ default: ++ return bpf_convert_ctx_access(type, si, insn_buf, prog, ++ target_size); ++ } ++ ++ return insn - insn_buf; ++} ++ ++static u32 xdp_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++/* SOCK_ADDR_LOAD_NESTED_FIELD() loads Nested Field S.F.NF where S is type of ++ * context Structure, F is Field in context structure that contains a pointer ++ * to Nested Structure of type NS that has the field NF. ++ * ++ * SIZE encodes the load size (BPF_B, BPF_H, etc). It's up to caller to make ++ * sure that SIZE is not greater than actual size of S.F.NF. ++ * ++ * If offset OFF is provided, the load happens from that offset relative to ++ * offset of NF. ++ */ ++#define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF) \ ++ do { \ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg, \ ++ si->src_reg, offsetof(S, F)); \ ++ *insn++ = BPF_LDX_MEM( \ ++ SIZE, si->dst_reg, si->dst_reg, \ ++ bpf_target_off(NS, NF, FIELD_SIZEOF(NS, NF), \ ++ target_size) \ ++ + OFF); \ ++ } while (0) ++ ++#define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF) \ ++ SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, \ ++ BPF_FIELD_SIZEOF(NS, NF), 0) ++ ++/* SOCK_ADDR_STORE_NESTED_FIELD_OFF() has semantic similar to ++ * SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF() but for store operation. ++ * ++ * In addition it uses Temporary Field TF (member of struct S) as the 3rd ++ * "register" since two registers available in convert_ctx_access are not ++ * enough: we can't override neither SRC, since it contains value to store, nor ++ * DST since it contains pointer to context that may be used by later ++ * instructions. But we need a temporary place to save pointer to nested ++ * structure whose field we want to store to. ++ */ ++#define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF) \ ++ do { \ ++ int tmp_reg = BPF_REG_9; \ ++ if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \ ++ --tmp_reg; \ ++ if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \ ++ --tmp_reg; \ ++ *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg, \ ++ offsetof(S, TF)); \ ++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg, \ ++ si->dst_reg, offsetof(S, F)); \ ++ *insn++ = BPF_STX_MEM(SIZE, tmp_reg, si->src_reg, \ ++ bpf_target_off(NS, NF, FIELD_SIZEOF(NS, NF), \ ++ target_size) \ ++ + OFF); \ ++ *insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg, \ ++ offsetof(S, TF)); \ ++ } while (0) ++ ++#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \ ++ TF) \ ++ do { \ ++ if (type == BPF_WRITE) { \ ++ SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, \ ++ OFF, TF); \ ++ } else { \ ++ SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF( \ ++ S, NS, F, NF, SIZE, OFF); \ ++ } \ ++ } while (0) ++ ++#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD(S, NS, F, NF, TF) \ ++ SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF( \ ++ S, NS, F, NF, BPF_FIELD_SIZEOF(NS, NF), 0, TF) ++ ++static u32 sock_addr_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, ++ u32 *target_size) ++{ ++ return 0; ++} ++ ++static u32 sk_skb_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++static u32 sk_msg_convert_ctx_access(enum bpf_access_type type, ++ const struct bpf_insn *si, ++ struct bpf_insn *insn_buf, ++ struct bpf_prog *prog, u32 *target_size) ++{ ++ return 0; ++} ++ ++const struct bpf_verifier_ops sk_filter_verifier_ops = { ++ .is_valid_access = sk_filter_is_valid_access, ++ .convert_ctx_access = bpf_convert_ctx_access, ++ .gen_ld_abs = bpf_gen_ld_abs, + }; + +-static const struct bpf_verifier_ops tc_cls_act_ops = { +- .get_func_proto = tc_cls_act_func_proto, +- .is_valid_access = tc_cls_act_is_valid_access, +- .convert_ctx_access = bpf_net_convert_ctx_access, ++const struct bpf_prog_ops sk_filter_prog_ops = { + }; + +-static struct bpf_prog_type_list sk_filter_type __read_mostly = { +- .ops = &sk_filter_ops, +- .type = BPF_PROG_TYPE_SOCKET_FILTER, ++const struct bpf_verifier_ops tc_cls_act_verifier_ops = { ++ .get_func_proto = tc_cls_act_func_proto, ++ .is_valid_access = tc_cls_act_is_valid_access, ++ .convert_ctx_access = tc_cls_act_convert_ctx_access, ++ .gen_prologue = tc_cls_act_prologue, ++ .gen_ld_abs = bpf_gen_ld_abs, + }; + +-static struct bpf_prog_type_list sched_cls_type __read_mostly = { +- .ops = &tc_cls_act_ops, +- .type = BPF_PROG_TYPE_SCHED_CLS, ++const struct bpf_prog_ops tc_cls_act_prog_ops = { + }; + +-static struct bpf_prog_type_list sched_act_type __read_mostly = { +- .ops = &tc_cls_act_ops, +- .type = BPF_PROG_TYPE_SCHED_ACT, ++const struct bpf_verifier_ops xdp_verifier_ops = { ++ .get_func_proto = xdp_func_proto, ++ .is_valid_access = xdp_is_valid_access, ++ .convert_ctx_access = xdp_convert_ctx_access, ++ .gen_prologue = bpf_noop_prologue, + }; + +-static int __init register_sk_filter_ops(void) +-{ +- bpf_register_prog_type(&sk_filter_type); +- bpf_register_prog_type(&sched_cls_type); +- bpf_register_prog_type(&sched_act_type); ++const struct bpf_verifier_ops lwt_in_verifier_ops = { ++ .get_func_proto = lwt_in_func_proto, ++ .is_valid_access = lwt_is_valid_access, ++ .convert_ctx_access = bpf_convert_ctx_access, ++}; + +- return 0; +-} +-late_initcall(register_sk_filter_ops); ++const struct bpf_prog_ops lwt_in_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops lwt_out_verifier_ops = { ++ .get_func_proto = lwt_out_func_proto, ++ .is_valid_access = lwt_is_valid_access, ++ .convert_ctx_access = bpf_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops lwt_out_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops lwt_xmit_verifier_ops = { ++ .get_func_proto = lwt_xmit_func_proto, ++ .is_valid_access = lwt_is_valid_access, ++ .convert_ctx_access = bpf_convert_ctx_access, ++ .gen_prologue = tc_cls_act_prologue, ++}; ++ ++const struct bpf_prog_ops lwt_xmit_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops lwt_seg6local_verifier_ops = { ++ .get_func_proto = lwt_seg6local_func_proto, ++ .is_valid_access = lwt_is_valid_access, ++ .convert_ctx_access = bpf_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops lwt_seg6local_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops cg_sock_verifier_ops = { ++ .get_func_proto = sock_filter_func_proto, ++ .is_valid_access = sock_filter_is_valid_access, ++ .convert_ctx_access = bpf_sock_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops cg_sock_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops cg_sock_addr_verifier_ops = { ++ .get_func_proto = sock_addr_func_proto, ++ .is_valid_access = sock_addr_is_valid_access, ++ .convert_ctx_access = sock_addr_convert_ctx_access, ++}; + +-int __sk_detach_filter(struct sock *sk, bool locked) ++const struct bpf_prog_ops cg_sock_addr_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops sock_ops_verifier_ops = { ++ .get_func_proto = sock_ops_func_proto, ++ .is_valid_access = sock_ops_is_valid_access, ++ .convert_ctx_access = sock_ops_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops sock_ops_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops sk_skb_verifier_ops = { ++ .get_func_proto = sk_skb_func_proto, ++ .is_valid_access = sk_skb_is_valid_access, ++ .convert_ctx_access = sk_skb_convert_ctx_access, ++ .gen_prologue = sk_skb_prologue, ++}; ++ ++const struct bpf_prog_ops sk_skb_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops sk_msg_verifier_ops = { ++ .get_func_proto = sk_msg_func_proto, ++ .is_valid_access = sk_msg_is_valid_access, ++ .convert_ctx_access = sk_msg_convert_ctx_access, ++ .gen_prologue = bpf_noop_prologue, ++}; ++ ++const struct bpf_prog_ops sk_msg_prog_ops = { ++}; ++ ++const struct bpf_verifier_ops flow_dissector_verifier_ops = { ++ .get_func_proto = flow_dissector_func_proto, ++ .is_valid_access = flow_dissector_is_valid_access, ++ .convert_ctx_access = flow_dissector_convert_ctx_access, ++}; ++ ++const struct bpf_prog_ops flow_dissector_prog_ops = { ++}; ++ ++int sk_detach_filter(struct sock *sk) + { + int ret = -ENOENT; + struct sk_filter *filter; +@@ -1928,7 +3664,8 @@ int __sk_detach_filter(struct sock *sk, + if (sock_flag(sk, SOCK_FILTER_LOCKED)) + return -EPERM; + +- filter = rcu_dereference_protected(sk->sk_filter, locked); ++ filter = rcu_dereference_protected(sk->sk_filter, ++ lockdep_sock_is_held(sk)); + if (filter) { + RCU_INIT_POINTER(sk->sk_filter, NULL); + sk_filter_uncharge(sk, filter); +@@ -1937,12 +3674,7 @@ int __sk_detach_filter(struct sock *sk, + + return ret; + } +-EXPORT_SYMBOL_GPL(__sk_detach_filter); +- +-int sk_detach_filter(struct sock *sk) +-{ +- return __sk_detach_filter(sk, sock_owned_by_user(sk)); +-} ++EXPORT_SYMBOL_GPL(sk_detach_filter); + + int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, + unsigned int len) +@@ -1953,7 +3685,7 @@ int sk_get_filter(struct sock *sk, struc + + lock_sock(sk); + filter = rcu_dereference_protected(sk->sk_filter, +- sock_owned_by_user(sk)); ++ lockdep_sock_is_held(sk)); + if (!filter) + goto out; + +@@ -1987,3 +3719,5 @@ out: + release_sock(sk); + return ret; + } ++ ++ +--- a/include/asm-generic/barrier.h ++++ b/include/asm-generic/barrier.h +@@ -119,5 +119,29 @@ do { \ + ___p1; \ + }) + ++/** ++ * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees ++ * @ptr: pointer to the variable to wait on ++ * @cond: boolean expression to wait for ++ * ++ * Equivalent to using READ_ONCE() on the condition variable. ++ * ++ * Due to C lacking lambda expressions we load the value of *ptr into a ++ * pre-named variable @VAL to be used in @cond. ++ */ ++#ifndef smp_cond_load_relaxed ++#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ ++ typeof(ptr) __PTR = (ptr); \ ++ typeof(*ptr) VAL; \ ++ for (;;) { \ ++ VAL = READ_ONCE(*__PTR); \ ++ if (cond_expr) \ ++ break; \ ++ cpu_relax(); \ ++ } \ ++ VAL; \ ++}) ++#endif ++ + #endif /* !__ASSEMBLY__ */ + #endif /* __ASM_GENERIC_BARRIER_H */ +--- a/arch/arm/include/asm/barrier.h ++++ b/arch/arm/include/asm/barrier.h +@@ -94,4 +94,6 @@ do { \ + #define smp_mb__after_atomic() smp_mb() + + #endif /* !__ASSEMBLY__ */ ++ ++#include + #endif /* __ASM_BARRIER_H */ +--- a/include/linux/list_nulls.h ++++ b/include/linux/list_nulls.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ + #ifndef _LINUX_LIST_NULLS_H + #define _LINUX_LIST_NULLS_H + +@@ -29,6 +30,11 @@ struct hlist_nulls_node { + ((ptr)->first = (struct hlist_nulls_node *) NULLS_MARKER(nulls)) + + #define hlist_nulls_entry(ptr, type, member) container_of(ptr,type,member) ++ ++#define hlist_nulls_entry_safe(ptr, type, member) \ ++ ({ typeof(ptr) ____ptr = (ptr); \ ++ !is_a_nulls(____ptr) ? hlist_nulls_entry(____ptr, type, member) : NULL; \ ++ }) + /** + * ptr_is_a_nulls - Test if a ptr is a nulls + * @ptr: ptr to be tested +@@ -57,7 +63,7 @@ static inline int hlist_nulls_unhashed(c + + static inline int hlist_nulls_empty(const struct hlist_nulls_head *h) + { +- return is_a_nulls(h->first); ++ return is_a_nulls(READ_ONCE(h->first)); + } + + static inline void hlist_nulls_add_head(struct hlist_nulls_node *n, +@@ -66,10 +72,10 @@ static inline void hlist_nulls_add_head( + struct hlist_nulls_node *first = h->first; + + n->next = first; +- n->pprev = &h->first; ++ WRITE_ONCE(n->pprev, &h->first); + h->first = n; + if (!is_a_nulls(first)) +- first->pprev = &n->next; ++ WRITE_ONCE(first->pprev, &n->next); + } + + static inline void __hlist_nulls_del(struct hlist_nulls_node *n) +@@ -79,13 +85,13 @@ static inline void __hlist_nulls_del(str + + WRITE_ONCE(*pprev, next); + if (!is_a_nulls(next)) +- next->pprev = pprev; ++ WRITE_ONCE(next->pprev, pprev); + } + + static inline void hlist_nulls_del(struct hlist_nulls_node *n) + { + __hlist_nulls_del(n); +- n->pprev = LIST_POISON2; ++ WRITE_ONCE(n->pprev, LIST_POISON2); + } + + /** +--- a/include/linux/rculist_nulls.h ++++ b/include/linux/rculist_nulls.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ + #ifndef _LINUX_RCULIST_NULLS_H + #define _LINUX_RCULIST_NULLS_H + +@@ -33,7 +34,7 @@ static inline void hlist_nulls_del_init_ + { + if (!hlist_nulls_unhashed(n)) { + __hlist_nulls_del(n); +- n->pprev = NULL; ++ WRITE_ONCE(n->pprev, NULL); + } + } + +@@ -65,7 +66,7 @@ static inline void hlist_nulls_del_init_ + static inline void hlist_nulls_del_rcu(struct hlist_nulls_node *n) + { + __hlist_nulls_del(n); +- n->pprev = LIST_POISON2; ++ WRITE_ONCE(n->pprev, LIST_POISON2); + } + + /** +@@ -93,11 +94,49 @@ static inline void hlist_nulls_add_head_ + struct hlist_nulls_node *first = h->first; + + n->next = first; +- n->pprev = &h->first; ++ WRITE_ONCE(n->pprev, &h->first); + rcu_assign_pointer(hlist_nulls_first_rcu(h), n); + if (!is_a_nulls(first)) +- first->pprev = &n->next; ++ WRITE_ONCE(first->pprev, &n->next); + } ++ ++/** ++ * hlist_nulls_add_tail_rcu ++ * @n: the element to add to the hash list. ++ * @h: the list to add to. ++ * ++ * Description: ++ * Adds the specified element to the specified hlist_nulls, ++ * while permitting racing traversals. ++ * ++ * The caller must take whatever precautions are necessary ++ * (such as holding appropriate locks) to avoid racing ++ * with another list-mutation primitive, such as hlist_nulls_add_head_rcu() ++ * or hlist_nulls_del_rcu(), running on this same list. ++ * However, it is perfectly legal to run concurrently with ++ * the _rcu list-traversal primitives, such as ++ * hlist_nulls_for_each_entry_rcu(), used to prevent memory-consistency ++ * problems on Alpha CPUs. Regardless of the type of CPU, the ++ * list-traversal primitive must be guarded by rcu_read_lock(). ++ */ ++static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, ++ struct hlist_nulls_head *h) ++{ ++ struct hlist_nulls_node *i, *last = NULL; ++ ++ /* Note: write side code, so rcu accessors are not needed. */ ++ for (i = h->first; !is_a_nulls(i); i = i->next) ++ last = i; ++ ++ if (last) { ++ n->next = last->next; ++ n->pprev = &last->next; ++ rcu_assign_pointer(hlist_next_rcu(last), n); ++ } else { ++ hlist_nulls_add_head_rcu(n, h); ++ } ++} ++ + /** + * hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type + * @tpos: the type * to use as a loop cursor. +@@ -107,7 +146,7 @@ static inline void hlist_nulls_add_head_ + * + * The barrier() is needed to make sure compiler doesn't cache first element [1], + * as this loop can be restarted [2] +- * [1] Documentation/atomic_ops.txt around line 114 ++ * [1] Documentation/core-api/atomic_ops.rst around line 114 + * [2] Documentation/RCU/rculist_nulls.txt around line 146 + */ + #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ +@@ -117,5 +156,19 @@ static inline void hlist_nulls_add_head_ + ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ + pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos))) + ++/** ++ * hlist_nulls_for_each_entry_safe - ++ * iterate over list of given type safe against removal of list entry ++ * @tpos: the type * to use as a loop cursor. ++ * @pos: the &struct hlist_nulls_node to use as a loop cursor. ++ * @head: the head for your list. ++ * @member: the name of the hlist_nulls_node within the struct. ++ */ ++#define hlist_nulls_for_each_entry_safe(tpos, pos, head, member) \ ++ for (({barrier();}), \ ++ pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \ ++ (!is_a_nulls(pos)) && \ ++ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); \ ++ pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)); 1; });) + #endif + #endif +--- a/drivers/net/tun.c ++++ b/drivers/net/tun.c +@@ -627,8 +627,9 @@ static int tun_attach(struct tun_struct + + /* Re-attach the filter to persist device */ + if (!skip_filter && (tun->filter_attached == true)) { +- err = __sk_attach_filter(&tun->fprog, tfile->socket.sk, +- lockdep_rtnl_is_held()); ++ lock_sock(tfile->socket.sk); ++ err = sk_attach_filter(&tun->fprog, tfile->socket.sk); ++ release_sock(tfile->socket.sk); + if (!err) + goto out; + } +@@ -1835,7 +1836,9 @@ static void tun_detach_filter(struct tun + + for (i = 0; i < n; i++) { + tfile = rtnl_dereference(tun->tfiles[i]); +- __sk_detach_filter(tfile->socket.sk, lockdep_rtnl_is_held()); ++ lock_sock(tfile->socket.sk); ++ sk_detach_filter(tfile->socket.sk); ++ release_sock(tfile->socket.sk); + } + + tun->filter_attached = false; +@@ -1848,8 +1851,9 @@ static int tun_attach_filter(struct tun_ + + for (i = 0; i < tun->numqueues; i++) { + tfile = rtnl_dereference(tun->tfiles[i]); +- ret = __sk_attach_filter(&tun->fprog, tfile->socket.sk, +- lockdep_rtnl_is_held()); ++ lock_sock(tfile->socket.sk); ++ ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); ++ release_sock(tfile->socket.sk); + if (ret) { + tun_detach_filter(tun, i); + return ret; +--- a/include/linux/list.h ++++ b/include/linux/list.h +@@ -1,3 +1,4 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ + #ifndef _LINUX_LIST_H + #define _LINUX_LIST_H + +@@ -24,31 +25,46 @@ + + static inline void INIT_LIST_HEAD(struct list_head *list) + { +- list->next = list; ++ WRITE_ONCE(list->next, list); + list->prev = list; + } + ++#ifdef CONFIG_DEBUG_LIST ++extern bool __list_add_valid(struct list_head *new, ++ struct list_head *prev, ++ struct list_head *next); ++extern bool __list_del_entry_valid(struct list_head *entry); ++#else ++static inline bool __list_add_valid(struct list_head *new, ++ struct list_head *prev, ++ struct list_head *next) ++{ ++ return true; ++} ++static inline bool __list_del_entry_valid(struct list_head *entry) ++{ ++ return true; ++} ++#endif ++ + /* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +-#ifndef CONFIG_DEBUG_LIST + static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) + { ++ if (!__list_add_valid(new, prev, next)) ++ return; ++ + next->prev = new; + new->next = next; + new->prev = prev; +- prev->next = new; ++ WRITE_ONCE(prev->next, new); + } +-#else +-extern void __list_add(struct list_head *new, +- struct list_head *prev, +- struct list_head *next); +-#endif + + /** + * list_add - add a new entry +@@ -90,28 +106,40 @@ static inline void __list_del(struct lis + WRITE_ONCE(prev->next, next); + } + ++/* ++ * Delete a list entry and clear the 'prev' pointer. ++ * ++ * This is a special-purpose list clearing method used in the networking code ++ * for lists allocated as per-cpu, where we don't want to incur the extra ++ * WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this ++ * needs to check the node 'prev' pointer instead of calling list_empty(). ++ */ ++static inline void __list_del_clearprev(struct list_head *entry) ++{ ++ __list_del(entry->prev, entry->next); ++ entry->prev = NULL; ++} ++ + /** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty() on entry does not return true after this, the entry is + * in an undefined state. + */ +-#ifndef CONFIG_DEBUG_LIST + static inline void __list_del_entry(struct list_head *entry) + { ++ if (!__list_del_entry_valid(entry)) ++ return; ++ + __list_del(entry->prev, entry->next); + } + + static inline void list_del(struct list_head *entry) + { +- __list_del(entry->prev, entry->next); ++ __list_del_entry(entry); + entry->next = LIST_POISON1; + entry->prev = LIST_POISON2; + } +-#else +-extern void __list_del_entry(struct list_head *entry); +-extern void list_del(struct list_head *entry); +-#endif + + /** + * list_replace - replace old entry by new one +@@ -137,6 +165,23 @@ static inline void list_replace_init(str + } + + /** ++ * list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position ++ * @entry1: the location to place entry2 ++ * @entry2: the location to place entry1 ++ */ ++static inline void list_swap(struct list_head *entry1, ++ struct list_head *entry2) ++{ ++ struct list_head *pos = entry2->prev; ++ ++ list_del(entry2); ++ list_replace(entry1, entry2); ++ if (pos == entry1) ++ pos = entry2; ++ list_add(entry1, pos); ++} ++ ++/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +@@ -170,6 +215,40 @@ static inline void list_move_tail(struct + } + + /** ++ * list_bulk_move_tail - move a subsection of a list to its tail ++ * @head: the head that will follow our entry ++ * @first: first entry to move ++ * @last: last entry to move, can be the same as first ++ * ++ * Move all entries between @first and including @last before @head. ++ * All three entries must belong to the same linked list. ++ */ ++static inline void list_bulk_move_tail(struct list_head *head, ++ struct list_head *first, ++ struct list_head *last) ++{ ++ first->prev->next = last->next; ++ last->next->prev = first->prev; ++ ++ head->prev->next = first; ++ first->prev = head->prev; ++ ++ last->next = head; ++ head->prev = last; ++} ++ ++/** ++ * list_is_first -- tests whether @list is the first entry in list @head ++ * @list: the entry to test ++ * @head: the head of the list ++ */ ++static inline int list_is_first(const struct list_head *list, ++ const struct list_head *head) ++{ ++ return list->prev == head; ++} ++ ++/** + * list_is_last - tests whether @list is the last entry in list @head + * @list: the entry to test + * @head: the head of the list +@@ -186,7 +265,7 @@ static inline int list_is_last(const str + */ + static inline int list_empty(const struct list_head *head) + { +- return head->next == head; ++ return READ_ONCE(head->next) == head; + } + + /** +@@ -223,6 +302,24 @@ static inline void list_rotate_left(stru + } + + /** ++ * list_rotate_to_front() - Rotate list to specific item. ++ * @list: The desired new front of the list. ++ * @head: The head of the list. ++ * ++ * Rotates list so that @list becomes the new front of the list. ++ */ ++static inline void list_rotate_to_front(struct list_head *list, ++ struct list_head *head) ++{ ++ /* ++ * Deletes the list head from the list denoted by @head and ++ * places it as the tail of @list, this effectively rotates the ++ * list so that @list is at the front. ++ */ ++ list_move_tail(head, list); ++} ++ ++/** + * list_is_singular - tests whether a list has just one entry. + * @head: the list to test. + */ +@@ -271,6 +368,36 @@ static inline void list_cut_position(str + __list_cut_position(list, head, entry); + } + ++/** ++ * list_cut_before - cut a list into two, before given entry ++ * @list: a new list to add all removed entries ++ * @head: a list with entries ++ * @entry: an entry within head, could be the head itself ++ * ++ * This helper moves the initial part of @head, up to but ++ * excluding @entry, from @head to @list. You should pass ++ * in @entry an element you know is on @head. @list should ++ * be an empty list or a list you do not care about losing ++ * its data. ++ * If @entry == @head, all entries on @head are moved to ++ * @list. ++ */ ++static inline void list_cut_before(struct list_head *list, ++ struct list_head *head, ++ struct list_head *entry) ++{ ++ if (head->next == entry) { ++ INIT_LIST_HEAD(list); ++ return; ++ } ++ list->next = head->next; ++ list->next->prev = list; ++ list->prev = entry->prev; ++ list->prev->next = list; ++ head->next = entry; ++ entry->prev = head; ++} ++ + static inline void __list_splice(const struct list_head *list, + struct list_head *prev, + struct list_head *next) +@@ -381,8 +508,11 @@ static inline void list_splice_tail_init + * + * Note that if the list is empty, it returns NULL. + */ +-#define list_first_entry_or_null(ptr, type, member) \ +- (!list_empty(ptr) ? list_first_entry(ptr, type, member) : NULL) ++#define list_first_entry_or_null(ptr, type, member) ({ \ ++ struct list_head *head__ = (ptr); \ ++ struct list_head *pos__ = READ_ONCE(head__->next); \ ++ pos__ != head__ ? list_entry(pos__, type, member) : NULL; \ ++}) + + /** + * list_next_entry - get the next element in list +@@ -511,6 +641,19 @@ static inline void list_splice_tail_init + pos = list_next_entry(pos, member)) + + /** ++ * list_for_each_entry_from_reverse - iterate backwards over list of given type ++ * from the current point ++ * @pos: the type * to use as a loop cursor. ++ * @head: the head for your list. ++ * @member: the name of the list_head within the struct. ++ * ++ * Iterate backwards over list of given type, continuing from current position. ++ */ ++#define list_for_each_entry_from_reverse(pos, head, member) \ ++ for (; &pos->member != (head); \ ++ pos = list_prev_entry(pos, member)) ++ ++/** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage +@@ -608,7 +751,7 @@ static inline int hlist_unhashed(const s + + static inline int hlist_empty(const struct hlist_head *h) + { +- return !h->first; ++ return !READ_ONCE(h->first); + } + + static inline void __hlist_del(struct hlist_node *n) +@@ -642,7 +785,7 @@ static inline void hlist_add_head(struct + n->next = first; + if (first) + first->pprev = &n->next; +- h->first = n; ++ WRITE_ONCE(h->first, n); + n->pprev = &h->first; + } + +@@ -653,7 +796,7 @@ static inline void hlist_add_before(stru + n->pprev = next->pprev; + n->next = next; + next->pprev = &n->next; +- *(n->pprev) = n; ++ WRITE_ONCE(*(n->pprev), n); + } + + static inline void hlist_add_behind(struct hlist_node *n, +@@ -679,6 +822,16 @@ static inline bool hlist_fake(struct hli + } + + /* ++ * Check whether the node is the only node of the head without ++ * accessing head: ++ */ ++static inline bool ++hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h) ++{ ++ return !n->next && n->pprev == &h->first; ++} ++ ++/* + * Move a list from one list head to another. Fixup the pprev + * reference of the first entry if it exists. + */ +--- /dev/null ++++ b/include/linux/ptr_ring.h +@@ -0,0 +1,673 @@ ++/* SPDX-License-Identifier: GPL-2.0-or-later */ ++/* ++ * Definitions for the 'struct ptr_ring' datastructure. ++ * ++ * Author: ++ * Michael S. Tsirkin ++ * ++ * Copyright (C) 2016 Red Hat, Inc. ++ * ++ * This is a limited-size FIFO maintaining pointers in FIFO order, with ++ * one CPU producing entries and another consuming entries from a FIFO. ++ * ++ * This implementation tries to minimize cache-contention when there is a ++ * single producer and a single consumer CPU. ++ */ ++ ++#ifndef _LINUX_PTR_RING_H ++#define _LINUX_PTR_RING_H 1 ++ ++#ifdef __KERNEL__ ++#include ++#include ++#include ++#include ++#include ++#include ++#endif ++ ++struct ptr_ring { ++ int producer ____cacheline_aligned_in_smp; ++ spinlock_t producer_lock; ++ int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */ ++ int consumer_tail; /* next entry to invalidate */ ++ spinlock_t consumer_lock; ++ /* Shared consumer/producer data */ ++ /* Read-only by both the producer and the consumer */ ++ int size ____cacheline_aligned_in_smp; /* max entries in queue */ ++ int batch; /* number of entries to consume in a batch */ ++ void **queue; ++}; ++ ++/* Note: callers invoking this in a loop must use a compiler barrier, ++ * for example cpu_relax(). ++ * ++ * NB: this is unlike __ptr_ring_empty in that callers must hold producer_lock: ++ * see e.g. ptr_ring_full. ++ */ ++static inline bool __ptr_ring_full(struct ptr_ring *r) ++{ ++ return r->queue[r->producer]; ++} ++ ++static inline bool ptr_ring_full(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock(&r->producer_lock); ++ ret = __ptr_ring_full(r); ++ spin_unlock(&r->producer_lock); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_full_irq(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock_irq(&r->producer_lock); ++ ret = __ptr_ring_full(r); ++ spin_unlock_irq(&r->producer_lock); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_full_any(struct ptr_ring *r) ++{ ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&r->producer_lock, flags); ++ ret = __ptr_ring_full(r); ++ spin_unlock_irqrestore(&r->producer_lock, flags); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_full_bh(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock_bh(&r->producer_lock); ++ ret = __ptr_ring_full(r); ++ spin_unlock_bh(&r->producer_lock); ++ ++ return ret; ++} ++ ++/* Note: callers invoking this in a loop must use a compiler barrier, ++ * for example cpu_relax(). Callers must hold producer_lock. ++ * Callers are responsible for making sure pointer that is being queued ++ * points to a valid data. ++ */ ++static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) ++{ ++ if (unlikely(!r->size) || r->queue[r->producer]) ++ return -ENOSPC; ++ ++ /* Make sure the pointer we are storing points to a valid data. */ ++ /* Pairs with smp_read_barrier_depends in __ptr_ring_consume. */ ++ smp_wmb(); ++ ++ WRITE_ONCE(r->queue[r->producer++], ptr); ++ if (unlikely(r->producer >= r->size)) ++ r->producer = 0; ++ return 0; ++} ++ ++/* ++ * Note: resize (below) nests producer lock within consumer lock, so if you ++ * consume in interrupt or BH context, you must disable interrupts/BH when ++ * calling this. ++ */ ++static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr) ++{ ++ int ret; ++ ++ spin_lock(&r->producer_lock); ++ ret = __ptr_ring_produce(r, ptr); ++ spin_unlock(&r->producer_lock); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr) ++{ ++ int ret; ++ ++ spin_lock_irq(&r->producer_lock); ++ ret = __ptr_ring_produce(r, ptr); ++ spin_unlock_irq(&r->producer_lock); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr) ++{ ++ unsigned long flags; ++ int ret; ++ ++ spin_lock_irqsave(&r->producer_lock, flags); ++ ret = __ptr_ring_produce(r, ptr); ++ spin_unlock_irqrestore(&r->producer_lock, flags); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr) ++{ ++ int ret; ++ ++ spin_lock_bh(&r->producer_lock); ++ ret = __ptr_ring_produce(r, ptr); ++ spin_unlock_bh(&r->producer_lock); ++ ++ return ret; ++} ++ ++static inline void *__ptr_ring_peek(struct ptr_ring *r) ++{ ++ if (likely(r->size)) ++ return READ_ONCE(r->queue[r->consumer_head]); ++ return NULL; ++} ++ ++/* ++ * Test ring empty status without taking any locks. ++ * ++ * NB: This is only safe to call if ring is never resized. ++ * ++ * However, if some other CPU consumes ring entries at the same time, the value ++ * returned is not guaranteed to be correct. ++ * ++ * In this case - to avoid incorrectly detecting the ring ++ * as empty - the CPU consuming the ring entries is responsible ++ * for either consuming all ring entries until the ring is empty, ++ * or synchronizing with some other CPU and causing it to ++ * re-test __ptr_ring_empty and/or consume the ring enteries ++ * after the synchronization point. ++ * ++ * Note: callers invoking this in a loop must use a compiler barrier, ++ * for example cpu_relax(). ++ */ ++static inline bool __ptr_ring_empty(struct ptr_ring *r) ++{ ++ if (likely(r->size)) ++ return !r->queue[READ_ONCE(r->consumer_head)]; ++ return true; ++} ++ ++static inline bool ptr_ring_empty(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock(&r->consumer_lock); ++ ret = __ptr_ring_empty(r); ++ spin_unlock(&r->consumer_lock); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_empty_irq(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock_irq(&r->consumer_lock); ++ ret = __ptr_ring_empty(r); ++ spin_unlock_irq(&r->consumer_lock); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_empty_any(struct ptr_ring *r) ++{ ++ unsigned long flags; ++ bool ret; ++ ++ spin_lock_irqsave(&r->consumer_lock, flags); ++ ret = __ptr_ring_empty(r); ++ spin_unlock_irqrestore(&r->consumer_lock, flags); ++ ++ return ret; ++} ++ ++static inline bool ptr_ring_empty_bh(struct ptr_ring *r) ++{ ++ bool ret; ++ ++ spin_lock_bh(&r->consumer_lock); ++ ret = __ptr_ring_empty(r); ++ spin_unlock_bh(&r->consumer_lock); ++ ++ return ret; ++} ++ ++/* Must only be called after __ptr_ring_peek returned !NULL */ ++static inline void __ptr_ring_discard_one(struct ptr_ring *r) ++{ ++ /* Fundamentally, what we want to do is update consumer ++ * index and zero out the entry so producer can reuse it. ++ * Doing it naively at each consume would be as simple as: ++ * consumer = r->consumer; ++ * r->queue[consumer++] = NULL; ++ * if (unlikely(consumer >= r->size)) ++ * consumer = 0; ++ * r->consumer = consumer; ++ * but that is suboptimal when the ring is full as producer is writing ++ * out new entries in the same cache line. Defer these updates until a ++ * batch of entries has been consumed. ++ */ ++ /* Note: we must keep consumer_head valid at all times for __ptr_ring_empty ++ * to work correctly. ++ */ ++ int consumer_head = r->consumer_head; ++ int head = consumer_head++; ++ ++ /* Once we have processed enough entries invalidate them in ++ * the ring all at once so producer can reuse their space in the ring. ++ * We also do this when we reach end of the ring - not mandatory ++ * but helps keep the implementation simple. ++ */ ++ if (unlikely(consumer_head - r->consumer_tail >= r->batch || ++ consumer_head >= r->size)) { ++ /* Zero out entries in the reverse order: this way we touch the ++ * cache line that producer might currently be reading the last; ++ * producer won't make progress and touch other cache lines ++ * besides the first one until we write out all entries. ++ */ ++ while (likely(head >= r->consumer_tail)) ++ r->queue[head--] = NULL; ++ r->consumer_tail = consumer_head; ++ } ++ if (unlikely(consumer_head >= r->size)) { ++ consumer_head = 0; ++ r->consumer_tail = 0; ++ } ++ /* matching READ_ONCE in __ptr_ring_empty for lockless tests */ ++ WRITE_ONCE(r->consumer_head, consumer_head); ++} ++ ++static inline void *__ptr_ring_consume(struct ptr_ring *r) ++{ ++ void *ptr; ++ ++ /* The READ_ONCE in __ptr_ring_peek guarantees that anyone ++ * accessing data through the pointer is up to date. Pairs ++ * with smp_wmb in __ptr_ring_produce. ++ */ ++ ptr = __ptr_ring_peek(r); ++ if (ptr) ++ __ptr_ring_discard_one(r); ++ ++ return ptr; ++} ++ ++static inline int __ptr_ring_consume_batched(struct ptr_ring *r, ++ void **array, int n) ++{ ++ void *ptr; ++ int i; ++ ++ for (i = 0; i < n; i++) { ++ ptr = __ptr_ring_consume(r); ++ if (!ptr) ++ break; ++ array[i] = ptr; ++ } ++ ++ return i; ++} ++ ++/* ++ * Note: resize (below) nests producer lock within consumer lock, so if you ++ * call this in interrupt or BH context, you must disable interrupts/BH when ++ * producing. ++ */ ++static inline void *ptr_ring_consume(struct ptr_ring *r) ++{ ++ void *ptr; ++ ++ spin_lock(&r->consumer_lock); ++ ptr = __ptr_ring_consume(r); ++ spin_unlock(&r->consumer_lock); ++ ++ return ptr; ++} ++ ++static inline void *ptr_ring_consume_irq(struct ptr_ring *r) ++{ ++ void *ptr; ++ ++ spin_lock_irq(&r->consumer_lock); ++ ptr = __ptr_ring_consume(r); ++ spin_unlock_irq(&r->consumer_lock); ++ ++ return ptr; ++} ++ ++static inline void *ptr_ring_consume_any(struct ptr_ring *r) ++{ ++ unsigned long flags; ++ void *ptr; ++ ++ spin_lock_irqsave(&r->consumer_lock, flags); ++ ptr = __ptr_ring_consume(r); ++ spin_unlock_irqrestore(&r->consumer_lock, flags); ++ ++ return ptr; ++} ++ ++static inline void *ptr_ring_consume_bh(struct ptr_ring *r) ++{ ++ void *ptr; ++ ++ spin_lock_bh(&r->consumer_lock); ++ ptr = __ptr_ring_consume(r); ++ spin_unlock_bh(&r->consumer_lock); ++ ++ return ptr; ++} ++ ++static inline int ptr_ring_consume_batched(struct ptr_ring *r, ++ void **array, int n) ++{ ++ int ret; ++ ++ spin_lock(&r->consumer_lock); ++ ret = __ptr_ring_consume_batched(r, array, n); ++ spin_unlock(&r->consumer_lock); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r, ++ void **array, int n) ++{ ++ int ret; ++ ++ spin_lock_irq(&r->consumer_lock); ++ ret = __ptr_ring_consume_batched(r, array, n); ++ spin_unlock_irq(&r->consumer_lock); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_consume_batched_any(struct ptr_ring *r, ++ void **array, int n) ++{ ++ unsigned long flags; ++ int ret; ++ ++ spin_lock_irqsave(&r->consumer_lock, flags); ++ ret = __ptr_ring_consume_batched(r, array, n); ++ spin_unlock_irqrestore(&r->consumer_lock, flags); ++ ++ return ret; ++} ++ ++static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, ++ void **array, int n) ++{ ++ int ret; ++ ++ spin_lock_bh(&r->consumer_lock); ++ ret = __ptr_ring_consume_batched(r, array, n); ++ spin_unlock_bh(&r->consumer_lock); ++ ++ return ret; ++} ++ ++/* Cast to structure type and call a function without discarding from FIFO. ++ * Function must return a value. ++ * Callers must take consumer_lock. ++ */ ++#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r))) ++ ++#define PTR_RING_PEEK_CALL(r, f) ({ \ ++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ ++ \ ++ spin_lock(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ ++ spin_unlock(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v; \ ++}) ++ ++#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \ ++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ ++ \ ++ spin_lock_irq(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ ++ spin_unlock_irq(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v; \ ++}) ++ ++#define PTR_RING_PEEK_CALL_BH(r, f) ({ \ ++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ ++ \ ++ spin_lock_bh(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ ++ spin_unlock_bh(&(r)->consumer_lock); \ ++ __PTR_RING_PEEK_CALL_v; \ ++}) ++ ++#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \ ++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ ++ unsigned long __PTR_RING_PEEK_CALL_f;\ ++ \ ++ spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ ++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ ++ spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ ++ __PTR_RING_PEEK_CALL_v; \ ++}) ++ ++/* Not all gfp_t flags (besides GFP_KERNEL) are allowed. See ++ * documentation for vmalloc for which of them are legal. ++ */ ++static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) ++{ ++ if (size > KMALLOC_MAX_SIZE / sizeof(void *)) ++ return NULL; ++ return kmalloc_array(size, sizeof(void *), gfp | __GFP_ZERO); ++} ++ ++static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) ++{ ++ r->size = size; ++ r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue)); ++ /* We need to set batch at least to 1 to make logic ++ * in __ptr_ring_discard_one work correctly. ++ * Batching too much (because ring is small) would cause a lot of ++ * burstiness. Needs tuning, for now disable batching. ++ */ ++ if (r->batch > r->size / 2 || !r->batch) ++ r->batch = 1; ++} ++ ++static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) ++{ ++ r->queue = __ptr_ring_init_queue_alloc(size, gfp); ++ if (!r->queue) ++ return -ENOMEM; ++ ++ __ptr_ring_set_size(r, size); ++ r->producer = r->consumer_head = r->consumer_tail = 0; ++ spin_lock_init(&r->producer_lock); ++ spin_lock_init(&r->consumer_lock); ++ ++ return 0; ++} ++ ++/* ++ * Return entries into ring. Destroy entries that don't fit. ++ * ++ * Note: this is expected to be a rare slow path operation. ++ * ++ * Note: producer lock is nested within consumer lock, so if you ++ * resize you must make sure all uses nest correctly. ++ * In particular if you consume ring in interrupt or BH context, you must ++ * disable interrupts/BH when doing so. ++ */ ++static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n, ++ void (*destroy)(void *)) ++{ ++ unsigned long flags; ++ int head; ++ ++ spin_lock_irqsave(&r->consumer_lock, flags); ++ spin_lock(&r->producer_lock); ++ ++ if (!r->size) ++ goto done; ++ ++ /* ++ * Clean out buffered entries (for simplicity). This way following code ++ * can test entries for NULL and if not assume they are valid. ++ */ ++ head = r->consumer_head - 1; ++ while (likely(head >= r->consumer_tail)) ++ r->queue[head--] = NULL; ++ r->consumer_tail = r->consumer_head; ++ ++ /* ++ * Go over entries in batch, start moving head back and copy entries. ++ * Stop when we run into previously unconsumed entries. ++ */ ++ while (n) { ++ head = r->consumer_head - 1; ++ if (head < 0) ++ head = r->size - 1; ++ if (r->queue[head]) { ++ /* This batch entry will have to be destroyed. */ ++ goto done; ++ } ++ r->queue[head] = batch[--n]; ++ r->consumer_tail = head; ++ /* matching READ_ONCE in __ptr_ring_empty for lockless tests */ ++ WRITE_ONCE(r->consumer_head, head); ++ } ++ ++done: ++ /* Destroy all entries left in the batch. */ ++ while (n) ++ destroy(batch[--n]); ++ spin_unlock(&r->producer_lock); ++ spin_unlock_irqrestore(&r->consumer_lock, flags); ++} ++ ++static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, ++ int size, gfp_t gfp, ++ void (*destroy)(void *)) ++{ ++ int producer = 0; ++ void **old; ++ void *ptr; ++ ++ while ((ptr = __ptr_ring_consume(r))) ++ if (producer < size) ++ queue[producer++] = ptr; ++ else if (destroy) ++ destroy(ptr); ++ ++ if (producer >= size) ++ producer = 0; ++ __ptr_ring_set_size(r, size); ++ r->producer = producer; ++ r->consumer_head = 0; ++ r->consumer_tail = 0; ++ old = r->queue; ++ r->queue = queue; ++ ++ return old; ++} ++ ++/* ++ * Note: producer lock is nested within consumer lock, so if you ++ * resize you must make sure all uses nest correctly. ++ * In particular if you consume ring in interrupt or BH context, you must ++ * disable interrupts/BH when doing so. ++ */ ++static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, ++ void (*destroy)(void *)) ++{ ++ unsigned long flags; ++ void **queue = __ptr_ring_init_queue_alloc(size, gfp); ++ void **old; ++ ++ if (!queue) ++ return -ENOMEM; ++ ++ spin_lock_irqsave(&(r)->consumer_lock, flags); ++ spin_lock(&(r)->producer_lock); ++ ++ old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy); ++ ++ spin_unlock(&(r)->producer_lock); ++ spin_unlock_irqrestore(&(r)->consumer_lock, flags); ++ ++ kvfree(old); ++ ++ return 0; ++} ++ ++/* ++ * Note: producer lock is nested within consumer lock, so if you ++ * resize you must make sure all uses nest correctly. ++ * In particular if you consume ring in interrupt or BH context, you must ++ * disable interrupts/BH when doing so. ++ */ ++static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, ++ unsigned int nrings, ++ int size, ++ gfp_t gfp, void (*destroy)(void *)) ++{ ++ unsigned long flags; ++ void ***queues; ++ int i; ++ ++ queues = kmalloc_array(nrings, sizeof(*queues), gfp); ++ if (!queues) ++ goto noqueues; ++ ++ for (i = 0; i < nrings; ++i) { ++ queues[i] = __ptr_ring_init_queue_alloc(size, gfp); ++ if (!queues[i]) ++ goto nomem; ++ } ++ ++ for (i = 0; i < nrings; ++i) { ++ spin_lock_irqsave(&(rings[i])->consumer_lock, flags); ++ spin_lock(&(rings[i])->producer_lock); ++ queues[i] = __ptr_ring_swap_queue(rings[i], queues[i], ++ size, gfp, destroy); ++ spin_unlock(&(rings[i])->producer_lock); ++ spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags); ++ } ++ ++ for (i = 0; i < nrings; ++i) ++ kvfree(queues[i]); ++ ++ kfree(queues); ++ ++ return 0; ++ ++nomem: ++ while (--i >= 0) ++ kvfree(queues[i]); ++ ++ kfree(queues); ++ ++noqueues: ++ return -ENOMEM; ++} ++ ++static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) ++{ ++ void *ptr; ++ ++ if (destroy) ++ while ((ptr = ptr_ring_consume(r))) ++ destroy(ptr); ++ kvfree(r->queue); ++} ++ ++#endif /* _LINUX_PTR_RING_H */ +--- a/include/linux/skbuff.h ++++ b/include/linux/skbuff.h +@@ -37,6 +37,7 @@ + #include + #include + #include ++#include + #include + + /* A. Checksumming of received packets by device. +@@ -592,13 +593,23 @@ struct sk_buff { + */ + kmemcheck_bitfield_begin(flags1); + __u16 queue_mapping; ++ ++/* if you move cloned around you also must adapt those constants */ ++#ifdef __BIG_ENDIAN_BITFIELD ++#define CLONED_MASK (1 << 7) ++#else ++#define CLONED_MASK 1 ++#endif ++#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset) ++ ++ __u8 __cloned_offset[0]; + __u8 cloned:1, + nohdr:1, + fclone:2, + peeked:1, + head_frag:1, +- xmit_more:1; +- /* one bit hole */ ++ xmit_more:1, ++ __unused:1; /* one bit hole */ + kmemcheck_bitfield_end(flags1); + + /* fields enclosed in headers_start/headers_end are copied +@@ -639,6 +650,14 @@ struct sk_buff { + __u8 csum_level:2; + __u8 csum_bad:1; + ++#ifdef __BIG_ENDIAN_BITFIELD ++#define PKT_VLAN_PRESENT_BIT 7 ++#else ++#define PKT_VLAN_PRESENT_BIT 0 ++#endif ++#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset) ++ __u8 __pkt_vlan_present_offset[0]; ++ __u8 vlan_present:1; + #ifdef CONFIG_IPV6_NDISC_NODETYPE + __u8 ndisc_nodetype:2; + #endif +@@ -647,7 +666,7 @@ struct sk_buff { + __u8 remcsum_offload:1; + __u8 gro_skip:1; + __u8 fast_forwarded:1; +- /* 1 or 3 bit hole */ ++ /* 0 or 2 bit hole */ + + #ifdef CONFIG_NET_SCHED + __u16 tc_index; /* traffic control index */ +@@ -805,6 +824,15 @@ static inline struct rtable *skb_rtable( + return (struct rtable *)skb_dst(skb); + } + ++/* For mangling skb->pkt_type from user space side from applications ++ * such as nft, tc, etc, we only allow a conservative subset of ++ * possible pkt_types to be set. ++*/ ++static inline bool skb_pkt_type_ok(u32 ptype) ++{ ++ return ptype <= PACKET_OTHERHOST; ++} ++ + void kfree_skb(struct sk_buff *skb); + void kfree_skb_list(struct sk_buff *segs); + void skb_tx_error(struct sk_buff *skb); +@@ -2127,6 +2155,11 @@ static inline unsigned char *skb_mac_hea + return skb->head + skb->mac_header; + } + ++static inline u32 skb_mac_header_len(const struct sk_buff *skb) ++{ ++ return skb->network_header - skb->mac_header; ++} ++ + static inline int skb_mac_header_was_set(const struct sk_buff *skb) + { + return skb->mac_header != (typeof(skb->mac_header))~0U; +@@ -2256,7 +2289,7 @@ static inline int pskb_network_may_pull( + + int ___pskb_trim(struct sk_buff *skb, unsigned int len); + +-static inline void __skb_trim(struct sk_buff *skb, unsigned int len) ++static inline void __skb_set_length(struct sk_buff *skb, unsigned int len) + { + if (unlikely(skb_is_nonlinear(skb))) { + WARN_ON(1); +@@ -2266,6 +2299,11 @@ static inline void __skb_trim(struct sk_ + skb_set_tail_pointer(skb, len); + } + ++static inline void __skb_trim(struct sk_buff *skb, unsigned int len) ++{ ++ __skb_set_length(skb, len); ++} ++ + void skb_trim(struct sk_buff *skb, unsigned int len); + + static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) +@@ -2318,6 +2356,20 @@ static inline struct sk_buff *skb_reduce + return skb; + } + ++static inline int __skb_grow(struct sk_buff *skb, unsigned int len) ++{ ++ unsigned int diff = len - skb->len; ++ ++ if (skb_tailroom(skb) < diff) { ++ int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), ++ GFP_ATOMIC); ++ if (ret) ++ return ret; ++ } ++ __skb_set_length(skb, len); ++ return 0; ++} ++ + /** + * skb_orphan - orphan a buffer + * @skb: buffer to orphan +@@ -2818,6 +2870,18 @@ static inline int skb_linearize_cow(stru + __skb_linearize(skb) : 0; + } + ++static __always_inline void ++__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, ++ unsigned int off) ++{ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ skb->csum = csum_block_sub(skb->csum, ++ csum_partial(start, len, 0), off); ++ else if (skb->ip_summed == CHECKSUM_PARTIAL && ++ skb_checksum_start_offset(skb) < 0) ++ skb->ip_summed = CHECKSUM_NONE; ++} ++ + /** + * skb_postpull_rcsum - update checksum for received skb after pull + * @skb: buffer to update +@@ -2828,36 +2892,38 @@ static inline int skb_linearize_cow(stru + * update the CHECKSUM_COMPLETE checksum, or set ip_summed to + * CHECKSUM_NONE so that it can be recomputed from scratch. + */ +- + static inline void skb_postpull_rcsum(struct sk_buff *skb, + const void *start, unsigned int len) + { +- if (skb->ip_summed == CHECKSUM_COMPLETE) +- skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); +- else if (skb->ip_summed == CHECKSUM_PARTIAL && +- skb_checksum_start_offset(skb) < 0) +- skb->ip_summed = CHECKSUM_NONE; ++ __skb_postpull_rcsum(skb, start, len, 0); + } + +-unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); ++static __always_inline void ++__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, ++ unsigned int off) ++{ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ skb->csum = csum_block_add(skb->csum, ++ csum_partial(start, len, 0), off); ++} + ++/** ++ * skb_postpush_rcsum - update checksum for received skb after push ++ * @skb: buffer to update ++ * @start: start of data after push ++ * @len: length of data pushed ++ * ++ * After doing a push on a received packet, you need to call this to ++ * update the CHECKSUM_COMPLETE checksum. ++ */ + static inline void skb_postpush_rcsum(struct sk_buff *skb, + const void *start, unsigned int len) + { +- /* For performing the reverse operation to skb_postpull_rcsum(), +- * we can instead of ... +- * +- * skb->csum = csum_add(skb->csum, csum_partial(start, len, 0)); +- * +- * ... just use this equivalent version here to save a few +- * instructions. Feeding csum of 0 in csum_partial() and later +- * on adding skb->csum is equivalent to feed skb->csum in the +- * first place. +- */ +- if (skb->ip_summed == CHECKSUM_COMPLETE) +- skb->csum = csum_partial(start, len, skb->csum); ++ __skb_postpush_rcsum(skb, start, len, 0); + } + ++unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); ++ + /** + * skb_push_rcsum - push skb and update receive checksum + * @skb: buffer to update +@@ -2901,6 +2967,21 @@ static inline int pskb_trim_rcsum(struct + #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode)) + #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode)) + ++static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len) ++{ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ skb->ip_summed = CHECKSUM_NONE; ++ __skb_trim(skb, len); ++ return 0; ++} ++ ++static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len) ++{ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ skb->ip_summed = CHECKSUM_NONE; ++ return __skb_grow(skb, len); ++} ++ + #define skb_queue_walk(queue, skb) \ + for (skb = (queue)->next; \ + skb != (struct sk_buff *)(queue); \ +@@ -3662,6 +3743,13 @@ static inline bool skb_is_gso_v6(const s + return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; + } + ++static inline void skb_gso_reset(struct sk_buff *skb) ++{ ++ skb_shinfo(skb)->gso_size = 0; ++ skb_shinfo(skb)->gso_segs = 0; ++ skb_shinfo(skb)->gso_type = 0; ++} ++ + void __skb_warn_lro_forwarding(const struct sk_buff *skb); + + static inline bool skb_warn_if_lro(const struct sk_buff *skb) +--- a/include/linux/if_arp.h ++++ b/include/linux/if_arp.h +@@ -44,4 +44,21 @@ static inline int arp_hdr_len(struct net + return sizeof(struct arphdr) + (dev->addr_len + sizeof(u32)) * 2; + } + } ++ ++static inline bool dev_is_mac_header_xmit(const struct net_device *dev) ++{ ++ switch (dev->type) { ++ case ARPHRD_TUNNEL: ++ case ARPHRD_TUNNEL6: ++ case ARPHRD_SIT: ++ case ARPHRD_IPGRE: ++ case ARPHRD_VOID: ++ case ARPHRD_NONE: ++ case ARPHRD_RAWIP: ++ return false; ++ default: ++ return true; ++ } ++} ++ + #endif /* _LINUX_IF_ARP_H */ +--- a/include/linux/if_vlan.h ++++ b/include/linux/if_vlan.h +@@ -66,7 +66,6 @@ static inline struct vlan_ethhdr *vlan_e + #define VLAN_PRIO_MASK 0xe000 /* Priority Code Point */ + #define VLAN_PRIO_SHIFT 13 + #define VLAN_CFI_MASK 0x1000 /* Canonical Format Indicator */ +-#define VLAN_TAG_PRESENT VLAN_CFI_MASK + #define VLAN_VID_MASK 0x0fff /* VLAN Identifier */ + #define VLAN_N_VID 4096 + +@@ -78,8 +77,8 @@ static inline bool is_vlan_dev(struct ne + return dev->priv_flags & IFF_802_1Q_VLAN; + } + +-#define skb_vlan_tag_present(__skb) ((__skb)->vlan_tci & VLAN_TAG_PRESENT) +-#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci & ~VLAN_TAG_PRESENT) ++#define skb_vlan_tag_present(__skb) ((__skb)->vlan_present) ++#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci) + #define skb_vlan_tag_get_id(__skb) ((__skb)->vlan_tci & VLAN_VID_MASK) + + /** +@@ -376,6 +375,31 @@ static inline struct sk_buff *vlan_inser + return skb; + } + ++/** ++ * __vlan_hwaccel_clear_tag - clear hardware accelerated VLAN info ++ * @skb: skbuff to clear ++ * ++ * Clears the VLAN information from @skb ++ */ ++static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb) ++{ ++ skb->vlan_present = 0; ++} ++ ++/** ++ * __vlan_hwaccel_copy_tag - copy hardware accelerated VLAN info from another skb ++ * @dst: skbuff to copy to ++ * @src: skbuff to copy from ++ * ++ * Copies VLAN information from @src to @dst (for branchless code) ++ */ ++static inline void __vlan_hwaccel_copy_tag(struct sk_buff *dst, const struct sk_buff *src) ++{ ++ dst->vlan_present = src->vlan_present; ++ dst->vlan_proto = src->vlan_proto; ++ dst->vlan_tci = src->vlan_tci; ++} ++ + /* + * __vlan_hwaccel_push_inside - pushes vlan tag to the payload + * @skb: skbuff to tag +@@ -390,7 +414,7 @@ static inline struct sk_buff *__vlan_hwa + skb = vlan_insert_tag_set_proto(skb, skb->vlan_proto, + skb_vlan_tag_get(skb)); + if (likely(skb)) +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + return skb; + } + /* +@@ -422,7 +446,8 @@ static inline void __vlan_hwaccel_put_ta + __be16 vlan_proto, u16 vlan_tci) + { + skb->vlan_proto = vlan_proto; +- skb->vlan_tci = VLAN_TAG_PRESENT | vlan_tci; ++ skb->vlan_tci = vlan_tci; ++ skb->vlan_present = 1; + } + + /** +--- a/include/net/checksum.h ++++ b/include/net/checksum.h +@@ -120,6 +120,11 @@ static inline __wsum csum_partial_ext(co + + #define CSUM_MANGLED_0 ((__force __sum16)0xffff) + ++static inline void csum_replace_by_diff(__sum16 *sum, __wsum diff) ++{ ++ *sum = csum_fold(csum_add(diff, ~csum_unfold(*sum))); ++} ++ + static inline void csum_replace4(__sum16 *sum, __be32 from, __be32 to) + { + __wsum tmp = csum_sub(~csum_unfold(*sum), (__force __wsum)from); +--- a/lib/test_bpf.c ++++ b/lib/test_bpf.c +@@ -38,6 +38,7 @@ + #define SKB_HASH 0x1234aaab + #define SKB_QUEUE_MAP 123 + #define SKB_VLAN_TCI 0xffff ++#define SKB_VLAN_PRESENT 1 + #define SKB_DEV_IFINDEX 577 + #define SKB_DEV_TYPE 588 + +@@ -691,8 +692,8 @@ static struct bpf_test tests[] = { + CLASSIC, + { }, + { +- { 1, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT }, +- { 10, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT } ++ { 1, SKB_VLAN_TCI }, ++ { 10, SKB_VLAN_TCI } + }, + }, + { +@@ -705,8 +706,8 @@ static struct bpf_test tests[] = { + CLASSIC, + { }, + { +- { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, +- { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } ++ { 1, SKB_VLAN_PRESENT }, ++ { 10, SKB_VLAN_PRESENT } + }, + }, + { +@@ -4432,8 +4433,8 @@ static struct bpf_test tests[] = { + CLASSIC, + { }, + { +- { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, +- { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } ++ { 1, SKB_VLAN_PRESENT }, ++ { 10, SKB_VLAN_PRESENT } + }, + .fill_helper = bpf_fill_maxinsns6, + }, +@@ -5144,6 +5145,7 @@ static struct sk_buff *populate_skb(char + skb->hash = SKB_HASH; + skb->queue_mapping = SKB_QUEUE_MAP; + skb->vlan_tci = SKB_VLAN_TCI; ++ skb->vlan_present = SKB_VLAN_PRESENT; + skb->dev = &dev; + skb->dev->ifindex = SKB_DEV_IFINDEX; + skb->dev->type = SKB_DEV_TYPE; +--- a/include/linux/netdevice.h ++++ b/include/linux/netdevice.h +@@ -3171,6 +3171,21 @@ int __dev_forward_skb(struct net_device + int dev_forward_skb(struct net_device *dev, struct sk_buff *skb); + bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb); + ++static __always_inline int ____dev_forward_skb(struct net_device *dev, ++ struct sk_buff *skb) ++{ ++ if (skb_orphan_frags(skb, GFP_ATOMIC) || ++ unlikely(!is_skb_forwardable(dev, skb))) { ++ atomic_long_inc(&dev->rx_dropped); ++ kfree_skb(skb); ++ return NET_RX_DROP; ++ } ++ ++ skb_scrub_packet(skb, true); ++ skb->priority = 0; ++ return 0; ++} ++ + extern int netdev_budget; + + /* Called by rtnetlink.c:rtnl_unlock() */ +--- a/net/openvswitch/actions.c ++++ b/net/openvswitch/actions.c +@@ -246,7 +246,7 @@ static int push_vlan(struct sk_buff *skb + else + key->eth.tci = vlan->vlan_tci; + return skb_vlan_push(skb, vlan->vlan_tpid, +- ntohs(vlan->vlan_tci) & ~VLAN_TAG_PRESENT); ++ ntohs(vlan->vlan_tci) & ~VLAN_CFI_MASK); + } + + /* 'src' is already properly masked. */ +--- a/net/openvswitch/flow.c ++++ b/net/openvswitch/flow.c +@@ -318,7 +318,7 @@ static int parse_vlan(struct sk_buff *sk + return -ENOMEM; + + qp = (struct qtag_prefix *) skb->data; +- key->eth.tci = qp->tci | htons(VLAN_TAG_PRESENT); ++ key->eth.tci = qp->tci | htons(VLAN_CFI_MASK); + __skb_pull(skb, sizeof(struct qtag_prefix)); + + return 0; +--- a/net/openvswitch/flow.h ++++ b/net/openvswitch/flow.h +@@ -69,7 +69,7 @@ struct sw_flow_key { + struct { + u8 src[ETH_ALEN]; /* Ethernet source address. */ + u8 dst[ETH_ALEN]; /* Ethernet destination address. */ +- __be16 tci; /* 0 if no VLAN, VLAN_TAG_PRESENT set otherwise. */ ++ __be16 tci; /* 0 if no VLAN, VLAN_CFI_MASK set otherwise. */ + __be16 type; /* Ethernet frame type. */ + } eth; + union { +--- a/net/openvswitch/flow_netlink.c ++++ b/net/openvswitch/flow_netlink.c +@@ -925,11 +925,11 @@ static int ovs_key_from_nlattrs(struct n + __be16 tci; + + tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]); +- if (!(tci & htons(VLAN_TAG_PRESENT))) { ++ if (!(tci & htons(VLAN_CFI_MASK))) { + if (is_mask) +- OVS_NLERR(log, "VLAN TCI mask does not have exact match for VLAN_TAG_PRESENT bit."); ++ OVS_NLERR(log, "VLAN TCI mask does not have exact match for VLAN_CFI_MASK bit."); + else +- OVS_NLERR(log, "VLAN TCI does not have VLAN_TAG_PRESENT bit set."); ++ OVS_NLERR(log, "VLAN TCI does not have VLAN_CFI_MASK bit set."); + + return -EINVAL; + } +@@ -1209,7 +1209,7 @@ int ovs_nla_get_match(struct net *net, s + key_attrs &= ~(1 << OVS_KEY_ATTR_ENCAP); + encap_valid = true; + +- if (tci & htons(VLAN_TAG_PRESENT)) { ++ if (tci & htons(VLAN_CFI_MASK)) { + err = parse_flow_nlattrs(encap, a, &key_attrs, log); + if (err) + return err; +@@ -1297,7 +1297,7 @@ int ovs_nla_get_match(struct net *net, s + if (a[OVS_KEY_ATTR_VLAN]) + tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]); + +- if (!(tci & htons(VLAN_TAG_PRESENT))) { ++ if (!(tci & htons(VLAN_CFI_MASK))) { + OVS_NLERR(log, "VLAN tag present bit must have an exact match (tci_mask=%x).", + ntohs(tci)); + err = -EINVAL; +@@ -2272,7 +2272,7 @@ static int __ovs_nla_copy_actions(struct + vlan = nla_data(a); + if (vlan->vlan_tpid != htons(ETH_P_8021Q)) + return -EINVAL; +- if (!(vlan->vlan_tci & htons(VLAN_TAG_PRESENT))) ++ if (!(vlan->vlan_tci & htons(VLAN_CFI_MASK))) + return -EINVAL; + vlan_tci = vlan->vlan_tci; + break; +@@ -2288,7 +2288,7 @@ static int __ovs_nla_copy_actions(struct + /* Prohibit push MPLS other than to a white list + * for packets that have a known tag order. + */ +- if (vlan_tci & htons(VLAN_TAG_PRESENT) || ++ if (vlan_tci & htons(VLAN_CFI_MASK) || + (eth_type != htons(ETH_P_IP) && + eth_type != htons(ETH_P_IPV6) && + eth_type != htons(ETH_P_ARP) && +@@ -2300,7 +2300,7 @@ static int __ovs_nla_copy_actions(struct + } + + case OVS_ACTION_ATTR_POP_MPLS: +- if (vlan_tci & htons(VLAN_TAG_PRESENT) || ++ if (vlan_tci & htons(VLAN_CFI_MASK) || + !eth_p_mpls(eth_type)) + return -EINVAL; + +--- a/net/sched/act_bpf.c ++++ b/net/sched/act_bpf.c +@@ -220,7 +220,7 @@ static int tcf_bpf_init_from_efd(struct + + bpf_fd = nla_get_u32(tb[TCA_ACT_BPF_FD]); + +- fp = bpf_prog_get(bpf_fd); ++ fp = bpf_prog_get_type_dev(bpf_fd, BPF_PROG_TYPE_SCHED_ACT, false); + if (IS_ERR(fp)) + return PTR_ERR(fp); + +--- a/net/sched/cls_bpf.c ++++ b/net/sched/cls_bpf.c +@@ -267,7 +267,7 @@ static int cls_bpf_prog_from_efd(struct + + bpf_fd = nla_get_u32(tb[TCA_BPF_FD]); + +- fp = bpf_prog_get(bpf_fd); ++ fp = bpf_prog_get_type_dev(bpf_fd, BPF_PROG_TYPE_SCHED_CLS, false); + if (IS_ERR(fp)) + return PTR_ERR(fp); + +--- a/net/8021q/vlan_core.c ++++ b/net/8021q/vlan_core.c +@@ -50,7 +50,7 @@ bool vlan_do_receive(struct sk_buff **sk + } + + skb->priority = vlan_get_ingress_priority(vlan_dev, skb->vlan_tci); +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + + rx_stats = this_cpu_ptr(vlan_dev_priv(vlan_dev)->vlan_pcpu_stats); + +--- a/net/ipv4/ip_tunnel_core.c ++++ b/net/ipv4/ip_tunnel_core.c +@@ -128,7 +128,7 @@ int iptunnel_pull_header(struct sk_buff + secpath_reset(skb); + skb_clear_hash_if_not_l4(skb); + skb_dst_drop(skb); +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + skb_set_queue_mapping(skb, 0); + skb->pkt_type = PACKET_HOST; + +--- a/net/bridge/br_netfilter_hooks.c ++++ b/net/bridge/br_netfilter_hooks.c +@@ -673,10 +673,8 @@ static int br_nf_push_frag_xmit(struct n + return 0; + } + +- if (data->vlan_tci) { +- skb->vlan_tci = data->vlan_tci; +- skb->vlan_proto = data->vlan_proto; +- } ++ if (data->vlan_proto) ++ __vlan_hwaccel_put_tag(skb, data->vlan_proto, data->vlan_tci); + + skb_copy_to_linear_data_offset(skb, -data->size, data->mac, data->size); + __skb_push(skb, data->encap_size); +@@ -740,8 +738,13 @@ static int br_nf_dev_queue_xmit(struct n + + data = this_cpu_ptr(&brnf_frag_data_storage); + +- data->vlan_tci = skb->vlan_tci; +- data->vlan_proto = skb->vlan_proto; ++ if (skb_vlan_tag_present(skb)) { ++ data->vlan_tci = skb->vlan_tci; ++ data->vlan_proto = skb->vlan_proto; ++ } else { ++ data->vlan_proto = 0; ++ } ++ + data->encap_size = nf_bridge_encap_header_len(skb); + data->size = ETH_HLEN + data->encap_size; + +--- a/net/bridge/br_private.h ++++ b/net/bridge/br_private.h +@@ -751,7 +751,7 @@ static inline int br_vlan_get_tag(const + int err = 0; + + if (skb_vlan_tag_present(skb)) { +- *vid = skb_vlan_tag_get(skb) & VLAN_VID_MASK; ++ *vid = skb_vlan_tag_get_id(skb); + } else { + *vid = 0; + err = -EINVAL; +--- a/net/bridge/br_vlan.c ++++ b/net/bridge/br_vlan.c +@@ -354,7 +354,7 @@ struct sk_buff *br_handle_vlan(struct ne + } + } + if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED) +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + + out: + return skb; +@@ -420,8 +420,8 @@ static bool __allowed_ingress(struct net + __vlan_hwaccel_put_tag(skb, proto, pvid); + else + /* Priority-tagged Frame. +- * At this point, We know that skb->vlan_tci had +- * VLAN_TAG_PRESENT bit and its VID field was 0x000. ++ * At this point, we know that skb->vlan_tci VID ++ * field was 0. + * We update only VID field and preserve PCP field. + */ + skb->vlan_tci |= pvid; +--- a/net/core/dev.c ++++ b/net/core/dev.c +@@ -3988,7 +3988,7 @@ ncls: + * and set skb->priority like in vlan_do_receive() + * For the time being, just ignore Priority Code Point + */ +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + } + + type = skb->protocol; +@@ -4211,7 +4211,9 @@ static void gro_list_prepare(struct napi + } + + diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; +- diffs |= p->vlan_tci ^ skb->vlan_tci; ++ diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); ++ if (skb_vlan_tag_present(p)) ++ diffs |= p->vlan_tci ^ skb->vlan_tci; + diffs |= skb_metadata_dst_cmp(p, skb); + if (maclen == ETH_HLEN) + diffs |= compare_ether_header(skb_mac_header(p), +@@ -4452,7 +4454,7 @@ static void napi_reuse_skb(struct napi_s + __skb_pull(skb, skb_headlen(skb)); + /* restore the reserve we had after netdev_alloc_skb_ip_align() */ + skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + skb->dev = napi->dev; + skb->skb_iif = 0; + skb->encapsulation = 0; +--- a/net/core/skbuff.c ++++ b/net/core/skbuff.c +@@ -4491,7 +4491,7 @@ int skb_vlan_pop(struct sk_buff *skb) + int err; + + if (likely(skb_vlan_tag_present(skb))) { +- skb->vlan_tci = 0; ++ __vlan_hwaccel_clear_tag(skb); + } else { + if (unlikely((skb->protocol != htons(ETH_P_8021Q) && + skb->protocol != htons(ETH_P_8021AD)) || +--- a/include/net/sock.h ++++ b/include/net/sock.h +@@ -1484,6 +1484,14 @@ do { \ + lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \ + } while (0) + ++#ifdef CONFIG_LOCKDEP ++static inline bool lockdep_sock_is_held(struct sock *sk) ++{ ++ return lockdep_is_held(&sk->sk_lock) || ++ lockdep_is_held(&sk->sk_lock.slock); ++} ++#endif ++ + void lock_sock_nested(struct sock *sk, int subclass); + + static inline void lock_sock(struct sock *sk) diff --git a/feeds/wifi-ax/mac80211/files/lib/wifi/mac80211.sh b/feeds/wifi-ax/mac80211/files/lib/wifi/mac80211.sh index 115e5b796..9b1f05a3d 100644 --- a/feeds/wifi-ax/mac80211/files/lib/wifi/mac80211.sh +++ b/feeds/wifi-ax/mac80211/files/lib/wifi/mac80211.sh @@ -178,6 +178,7 @@ detect_mac80211() { set wireless.radio${devidx}.channel=${channel} set wireless.radio${devidx}.band=${mode_band} set wireless.radio${devidx}.htmode=$htmode + set wireless.radio${devidx}.num_global_macaddr=8 ${scanning} set wireless.radio${devidx}.disabled=1 diff --git a/patches/0017-ipq807x-add-the-Qualcomm-AX-target-support.patch b/patches/0017-ipq807x-add-the-Qualcomm-AX-target-support.patch index 50c8e35e7..e23028309 100644 --- a/patches/0017-ipq807x-add-the-Qualcomm-AX-target-support.patch +++ b/patches/0017-ipq807x-add-the-Qualcomm-AX-target-support.patch @@ -1,7 +1,7 @@ -From 12003f391ccb3103f3e5590481f05384a8463ef1 Mon Sep 17 00:00:00 2001 +From c1670dcdda1ab0f5310f76e5ae6e710bd16892c7 Mon Sep 17 00:00:00 2001 From: John Crispin Date: Sat, 18 Jul 2020 08:53:44 +0200 -Subject: [PATCH 01/39] ipq807x: add the Qualcomm AX target support +Subject: [PATCH 01/38] ipq807x: add the Qualcomm AX target support Signed-off-by: John Crispin --- @@ -15,14 +15,13 @@ Signed-off-by: John Crispin .../ipq807x/base-files/etc/board.d/01_leds | 38 + .../ipq807x/base-files/etc/board.d/02_network | 81 + .../ipq807x/base-files/etc/board.d/03_wifi | 17 + - .../etc/hotplug.d/firmware/10-ath11k-caldata | 124 + + .../etc/hotplug.d/firmware/10-ath11k-caldata | 111 + .../etc/hotplug.d/firmware/11-ath10k-caldata | 16 + .../ipq807x/base-files/etc/hotplug.d/net/macs | 3 + .../ipq807x/base-files/etc/init.d/aq_phy | 21 + .../ipq807x/base-files/etc/init.d/bootcount | 12 + .../linux/ipq807x/base-files/etc/init.d/wdt | 14 + ...G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld | Bin 0 -> 391170 bytes - ...G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld | Bin 0 -> 393218 bytes .../base-files/lib/upgrade/platform.sh | 73 + target/linux/ipq807x/config-4.4 | 828 + .../arm/boot/dts/qcom-ipq6018-cig-wf188.dts | 18 + @@ -61,7 +60,7 @@ Signed-off-by: John Crispin .../ipq807x/patches/103-sercomm-wallaby.patch | 816 + target/linux/ipq807x/patches/104-wf194c.patch | 816 + .../patches/105-fix-dtc-gcc10-build.patch | 11 + - target/linux/ipq807x/patches/106-eap101.patch | 994 + + target/linux/ipq807x/patches/106-eap101.patch | 993 + .../linux/ipq807x/patches/108-log-spam.patch | 37 + target/linux/ipq807x/patches/109-tplink.patch | 1518 + .../ipq807x/patches/110-add-esmt-nand.patch | 37 + @@ -69,7 +68,7 @@ Signed-off-by: John Crispin target/linux/ipq807x/patches/112-pstore.patch | 147 + .../ipq807x/patches/200-bpf_backport.patch | 44780 ++++++++++++++++ toolchain/kernel-headers/Makefile | 8 + - 64 files changed, 56755 insertions(+), 2 deletions(-) + 63 files changed, 56741 insertions(+), 2 deletions(-) create mode 100644 package/boot/uboot-envtools/files/ipq807x create mode 100644 target/linux/ipq807x/109-logspam.patch create mode 100644 target/linux/ipq807x/Makefile @@ -83,7 +82,6 @@ Signed-off-by: John Crispin create mode 100755 target/linux/ipq807x/base-files/etc/init.d/bootcount create mode 100755 target/linux/ipq807x/base-files/etc/init.d/wdt create mode 100644 target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld - create mode 100644 target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld create mode 100755 target/linux/ipq807x/base-files/lib/upgrade/platform.sh create mode 100644 target/linux/ipq807x/config-4.4 create mode 100644 target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts @@ -246,65420 +244,6 @@ index b12c9af9a7..b7ee73b12e 100644 esac ;; *) -diff --git a/target/linux/ipq807x/109-logspam.patch b/target/linux/ipq807x/109-logspam.patch -new file mode 100644 -index 0000000000..79ee61dc77 ---- /dev/null -+++ b/target/linux/ipq807x/109-logspam.patch -@@ -0,0 +1,24 @@ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/clk/qcom/clk-branch.c -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c -+@@ -75,7 +75,7 @@ static int clk_branch_wait(const struct -+ bool (check_halt)(const struct clk_branch *, bool)) -+ { -+ bool voted = br->halt_check & BRANCH_VOTED; -+- const char *name = clk_hw_get_name(&br->clkr.hw); -++ //const char *name = clk_hw_get_name(&br->clkr.hw); -+ -+ /* Skip checking halt bit if the clock is in hardware gated mode */ -+ if (clk_branch_in_hwcg_mode(br)) -+@@ -93,8 +93,8 @@ static int clk_branch_wait(const struct -+ return 0; -+ udelay(1); -+ } -+- WARN(1, "%s status stuck at 'o%s'", name, -+- enabling ? "ff" : "n"); -++/* WARN(1, "%s status stuck at 'o%s'", name, -++ enabling ? "ff" : "n");*/ -+ return -EBUSY; -+ } -+ return 0; -diff --git a/target/linux/ipq807x/Makefile b/target/linux/ipq807x/Makefile -new file mode 100644 -index 0000000000..abdb82ec5d ---- /dev/null -+++ b/target/linux/ipq807x/Makefile -@@ -0,0 +1,22 @@ -+include $(TOPDIR)/rules.mk -+ -+ARCH:=arm -+BOARD:=ipq807x -+BOARDNAME:=Qualcomm Atheros AX -+SUBTARGETS:=ipq807x ipq60xx ipq50xx -+FEATURES:=squashfs ramdisk nand pcie usb -+KERNELNAME:=Image dtbs -+CPU_TYPE:=cortex-a7 -+ -+KERNEL_PATCHVER:=4.4 -+KERNEL_NAME_SUFFIX=-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016 -+ -+include $(INCLUDE_DIR)/target.mk -+DEFAULT_PACKAGES += kmod-qca-nss-dp kmod-qca-ssdk swconfig \ -+ kmod-qca-nss-drv \ -+ kmod-usb-phy-ipq807x kmod-usb-dwc3-of-simple \ -+ kmod-ath11k-ahb kmod-qrtr_mproc wpad \ -+ kmod-gpio-button-hotplug \ -+ qca-thermald-10.4 qca-ssdk-shell kmod-qca-nss-drv-bridge-mgr -+ -+$(eval $(call BuildTarget)) -diff --git a/target/linux/ipq807x/base-files/etc/board.d/01_leds b/target/linux/ipq807x/base-files/etc/board.d/01_leds -new file mode 100755 -index 0000000000..1f1797b0c6 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/board.d/01_leds -@@ -0,0 +1,38 @@ -+#!/bin/sh -+ -+. /lib/functions/uci-defaults.sh -+ -+board_config_update -+ -+board=$(board_name) -+ -+case "$board" in -+sercomm,wallaby|\ -+wallys,dr6018|\ -+cig,wf188n|\ -+cig,wf194c|\ -+cig,wf194c4) -+ ucidef_set_led_netdev "wan" "WAN" "green:wan" "eth0" "tx rx link" -+ ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" -+ ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" -+ ;; -+esac -+ -+case "$board" in -+edgecore,eap101) -+ ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" -+ ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" -+ ucidef_set_led_netdev "lan1" "lan1" "green:lan1" "eth1" -+ ucidef_set_led_netdev "lan2" "lan2" "green:lan2" "eth2" -+ ucidef_set_led_netdev "poe" "poe" "green:wan" "eth0" -+ ;; -+edgecore,eap102) -+ ucidef_set_led_netdev "poe" "poe" "green:wan" "eth0" -+ ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" -+ ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" -+ ;; -+esac -+ -+board_config_flush -+ -+exit 0 -diff --git a/target/linux/ipq807x/base-files/etc/board.d/02_network b/target/linux/ipq807x/base-files/etc/board.d/02_network -new file mode 100755 -index 0000000000..96bff27075 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/board.d/02_network -@@ -0,0 +1,81 @@ -+#!/bin/sh -+ -+. /lib/functions.sh -+. /lib/functions/uci-defaults.sh -+. /lib/functions/system.sh -+ -+qcom_setup_interfaces() -+{ -+ local board="$1" -+ ucidef_add_switch "switch0" -+ ucidef_add_switch_attr "switch0" "enable" "false" -+ ucidef_add_switch_attr "switch0" "reset" "false" -+ -+ case $board in -+ cig,wf188|\ -+ tplink,ex227|\ -+ tplink,ex447) -+ ucidef_set_interface_wan "eth0" -+ ucidef_set_interface_lan "" -+ ;; -+ qcom,ipq6018-cp01|\ -+ qcom,ipq807x-hk01) -+ ucidef_set_interface_lan "eth0 eth1 eth2 eth3 eth4" -+ ucidef_set_interface_wan "eth5" -+ ;; -+ cig,wf194c|\ -+ cig,wf194c4|\ -+ edgecore,eap106|\ -+ qcom,ipq5018-mp03.3|\ -+ sercomm,wallaby) -+ ucidef_set_interface_lan "eth0" -+ ucidef_set_interface_wan "eth1" -+ ;; -+ edgecore,eap101) -+ ucidef_set_interface_lan "eth1 eth2" -+ ucidef_set_interface_wan "eth0" -+ ;; -+ edgecore,eap102|\ -+ wallys,dr6018|\ -+ cig,wf188n) -+ ucidef_set_interface_lan "eth1" -+ ucidef_set_interface_wan "eth0" -+ ;; -+ qcom,ipq807x-hk14) -+ ucidef_set_interface_lan "eth0 eth1 eth2 eth3" -+ ucidef_set_interface_wan "eth4" -+ ;; -+ esac -+} -+ -+qcom_setup_macs() -+{ -+ local board="$1" -+ -+ case $board in -+ cig,wf194c|\ -+ cig,wf194c4) -+ mac=$(grep BaseMacAddress= /dev/mtd14 | cut -dx -f2) -+ wan_mac=$(macaddr_canonicalize $mac) -+ lan_mac=$(macaddr_add "$wan_mac" 1) -+ ucidef_set_network_device_mac eth0 $lan_mac -+ ucidef_set_network_device_mac eth1 $wan_mac -+ ucidef_set_label_macaddr $wan_mac -+ ;; -+ *) -+ wan_mac=$(cat /sys/class/net/eth0/address) -+ lan_mac=$(macaddr_add "$wan_mac" 1) -+ ;; -+ esac -+ [ -n "$lan_mac" ] && ucidef_set_interface_macaddr "lan" $lan_mac -+ [ -n "$wan_mac" ] && ucidef_set_interface_macaddr "wan" $wan_mac -+} -+ -+board_config_update -+board=$(board_name) -+ucidef_set_bridge_device bridge -+qcom_setup_interfaces $board -+qcom_setup_macs $board -+board_config_flush -+ -+exit 0 -diff --git a/target/linux/ipq807x/base-files/etc/board.d/03_wifi b/target/linux/ipq807x/base-files/etc/board.d/03_wifi -new file mode 100755 -index 0000000000..200e43d15c ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/board.d/03_wifi -@@ -0,0 +1,17 @@ -+#!/bin/sh -+ -+. /lib/functions/uci-defaults.sh -+ -+board_config_update -+ -+board=$(board_name) -+ -+case "$board" in -+edgecore,eap106) -+ ucidef_set_wifi_scanning 'soc/10000000.pci/pci0001:00/0001:00:00.0/0001:01:00.0' -+ ;; -+esac -+ -+board_config_flush -+ -+exit 0 -diff --git a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -new file mode 100755 -index 0000000000..7da120854e ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -@@ -0,0 +1,124 @@ -+#!/bin/sh -+ -+[ -e /lib/firmware/$FIRMWARE ] && exit 0 -+ -+. /lib/functions.sh -+. /lib/functions/system.sh -+ -+ath11k_generate_macs() { -+ touch /lib/firmware/ath11k-macs -+ eth=$(cat /sys/class/net/eth0/address) -+ mac1=$(macaddr_add $eth 2) -+ mac2=$(macaddr_add $eth 3) -+ mac3=$(macaddr_add $eth 4) -+ echo -ne \\x${mac1//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac2//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac3//:/\\x} >> /lib/firmware/ath11k-macs -+} -+ -+ath11k_generate_macs_wf194() { -+ touch /lib/firmware/ath11k-macs -+ mac=$(grep BaseMacAddress= /dev/mtd14 | cut -dx -f2) -+ eth=$(macaddr_canonicalize $mac) -+ mac1=$(macaddr_add $eth 2) -+ mac2=$(macaddr_add $eth 10) -+ mac3=$(macaddr_add $eth 18) -+ echo -ne \\x${mac1//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac2//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac3//:/\\x} >> /lib/firmware/ath11k-macs -+} -+ -+ath11k_generate_macs_eap101() { -+ touch /lib/firmware/ath11k-macs -+ eth=$(cat /sys/class/net/eth0/address) -+ mac1=$(macaddr_add $eth 4) -+ mac2=$(macaddr_add $eth 3) -+ mac3=$(macaddr_add $eth 5) -+ echo -ne \\x${mac1//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac2//:/\\x} >> /lib/firmware/ath11k-macs -+ echo -ne \\x${mac3//:/\\x} >> /lib/firmware/ath11k-macs -+} -+ -+caldata_die() { -+ echo "caldata: " "$*" -+ exit 1 -+} -+ -+caldata_extract() { -+ local part=$1 -+ local offset=$(($2)) -+ local count=$(($3)) -+ local mtd -+ -+ mtd=$(find_mtd_chardev $part) -+ [ -n "$mtd" ] || caldata_die "no mtd device found for partition $part" -+ -+ dd if=$mtd of=/lib/firmware/$FIRMWARE iflag=skip_bytes bs=$count skip=$offset count=1 2>/dev/null || \ -+ caldata_die "failed to extract calibration data from $mtd" -+} -+ -+board=$(board_name) -+ -+case "$FIRMWARE" in -+"ath11k/IPQ8074/hw2.0/caldata.bin") -+ case "$board" in -+ cig,wf194c|\ -+ cig,wf194c4|\ -+ edgecore,eap102 |\ -+ edgecore,eap106 |\ -+ qcom,ipq807x-hk01|\ -+ qcom,ipq807x-hk14|\ -+ tplink,ex227|\ -+ tplink,ex447|\ -+ sercomm,wallaby) -+ caldata_extract "0:ART" 0x1000 0x20000 -+ ;; -+ esac -+ ;; -+"ath11k/IPQ6018/hw1.0/caldata.bin") -+ case "$board" in -+ cig,wf188|\ -+ cig,wf188n|\ -+ edgecore,eap101|\ -+ wallys,dr6018|\ -+ qcom,ipq6018-cp01|\ -+ xiaomi,ax1800) -+ caldata_extract "0:ART" 0x1000 0x20000 -+ ;; -+ esac -+ ;; -+ath11k/QCN9074/hw1.0/caldata_1.bin) -+ case "$board" in -+ qcom,ipq807x-hk14) -+ caldata_extract "0:ART" 0x26800 0x20000 -+ ;; -+ esac -+ ;; -+ath11k/QCN9074/hw1.0/caldata_2.bin) -+ case "$board" in -+ qcom,ipq807x-hk14|\ -+ qcom,ipq5018-mp03.3) -+ caldata_extract "0:ART" 0x4C000 0x20000 -+ ;; -+ esac -+ ;; -+ath11k-macs) -+ case "$board" in -+ edgecore,eap101) -+ ath11k_generate_macs_eap101 -+ ;; -+ edgecore,eap102|\ -+ edgecore,eap106|\ -+ cig,wf188n) -+ ath11k_generate_macs -+ ;; -+ cig,wf194c|\ -+ cig,wf194c) -+ ath11k_generate_macs_wf194 -+ ;; -+ esac -+ ;; -+*) -+ exit 1 -+ ;; -+esac -diff --git a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata -new file mode 100644 -index 0000000000..eb23e8cd7a ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/11-ath10k-caldata -@@ -0,0 +1,16 @@ -+#!/bin/sh -+ -+[ -e /lib/firmware/$FIRMWARE ] && exit 0 -+ -+. /lib/functions/caldata.sh -+ -+board=$(board_name) -+ -+case "$FIRMWARE" in -+"ath10k/pre-cal-pci-0001:01:00.0.bin") -+ case $board in -+ edgecore,eap106) -+ caldata_extract "0:ART" 0x33000 0x2f20 -+ ;; -+ esac -+esac -diff --git a/target/linux/ipq807x/base-files/etc/hotplug.d/net/macs b/target/linux/ipq807x/base-files/etc/hotplug.d/net/macs -new file mode 100644 -index 0000000000..10f38108c8 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/hotplug.d/net/macs -@@ -0,0 +1,3 @@ -+#!/bin/sh -+mac=$(cat /etc/board.json | jsonfilter -e '@["network-device"]["'$DEVICENAME'"]'.macaddr) -+[ -n "$mac" ] && ip link set $DEVICENAME address $mac -diff --git a/target/linux/ipq807x/base-files/etc/init.d/aq_phy b/target/linux/ipq807x/base-files/etc/init.d/aq_phy -new file mode 100755 -index 0000000000..c56ce91ed1 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/init.d/aq_phy -@@ -0,0 +1,21 @@ -+#!/bin/sh /etc/rc.common -+ -+START=30 -+ -+boot() { -+ . /lib/functions.sh -+ -+ case "$(board_name)" in -+ cig,wf194c|\ -+ cig,wf194c4) -+ aq-fw-download /lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld miireg 0 > /dev/null -+ sleep 1 -+ ssdk_sh debug phy set 0 0x4004c441 0x8 -+ ;; -+ edgecore,eap106) -+ aq-fw-download /lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld miireg 8 > /dev/null -+ sleep 1 -+ ssdk_sh debug phy set 8 0x4004c441 0x8 -+ ;; -+ esac -+} -diff --git a/target/linux/ipq807x/base-files/etc/init.d/bootcount b/target/linux/ipq807x/base-files/etc/init.d/bootcount -new file mode 100755 -index 0000000000..a24f27353e ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/init.d/bootcount -@@ -0,0 +1,12 @@ -+#!/bin/sh /etc/rc.common -+ -+START=99 -+ -+boot() { -+ case "$(board_name)" in -+ edgecore,eap101|\ -+ edgecore,eap102) -+ fw_setenv bootcount 0 -+ ;; -+ esac -+} -diff --git a/target/linux/ipq807x/base-files/etc/init.d/wdt b/target/linux/ipq807x/base-files/etc/init.d/wdt -new file mode 100755 -index 0000000000..75de36a347 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/etc/init.d/wdt -@@ -0,0 +1,14 @@ -+#!/bin/sh /etc/rc.common -+ -+START=10 -+ -+boot() { -+ . /lib/functions.sh -+ -+ case "$(board_name)" in -+ cig,wf188|\ -+ cig,wf188n) -+ watchdog -t 5 /dev/watchdog1 -+ ;; -+ esac -+} -diff --git a/target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld b/target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.4.B-AQR_CIG_WIFI_ID44715_VER1673.cld -new file mode 100644 -index 0000000000000000000000000000000000000000..4a5a76cd34f390a2f6de2911cf91a74c42774602 -GIT binary patch -literal 391170 -zcmeFZdstJ~);GGcvPeiW#RM>N)kPqv2pB*SyfsitMYJ2Kq}6U)hz1pHYpuI#H*J$> -zDJtDNL|Z{^caoZNsnjI4-q1Egi#Kc`y?B9O2t+`U##HOw^^Fy@d%xc~@Atgt-RC>s -z`2!!Hm3bX=%- -zU>Jd61cnh9Mqn6$VFZQ|7)D?ifnfxO5g0~b7=d8~h7lM>U>Jd61cnh9Mqn6$VFZQ| -z7)D?ifnfxO5g0~b7=d8~h7lM>U>Jd61cnh9Mqn6$VFZQ|7)D?ifnfxO5g0~b7=d8~ -zh7lM>U>Jd61pdzcOlBQT7>FapB}3?nd% -zz%T;C2>kCuz@NnU7=evH#>8yQxo*agA8hF}|NC0}{~6`)a`->zknSAGtsHFfe}lL0 -zIQ*YEq?;qT#lfchH|n3x!P+?di#cR}j^u8RRGE{f$tl$3tTz1@**OiBIb=U)bsI-= -zmm~dvledFYSi@O;@W05;IbZvZ9P)aOR1_Y?EC14eJTQ -zt_ETHAWRn|A+f0YgJf?IS%gL18=QY{u;~6EV!ngt74ysd|7TXEOY$%ho9Xe8SYC?t -z!2aOXYcp0$CocQVnzjH9hFT*h?{W)m9V=z;JJ@elK9buR!2)hx4d4sUQ -zAi>$eq_cyPD|D+-XMjiPZ+Urd=1H?MtXYD*1sT=?!NjbI1vJx3^5Rnd-_52dmFs=o~VMtVSyQZulmHG^sVIn5Hgyrt4#+_X35MIlZg;DxdgNS!9 -z%wNB9P_<$ZD4+=2(^#a3CIIzS#e-TMwj+e6UovQ3I;fJ(ib3IVik_FA6oDiu -zgZzvX&hg*8OiHGxFp(aG=*q2SlQEr_PF_XFAn)Uy7?!L@cy=Q2NJ=W9 -zVA({>Qy|EjpCL7pN000-(A|1WM_Qo5vh*laS8dIX#&kD*bm-V)It%oOU_}BQX|+^H -z$y8OD48vsVN1lxW+4x6qBo3~gLN?DI!mucQFz-#tvZwbgd!|FO$(Xm@C|vImUc9-p -zTpv4Fm_q7O_=N?eu7JNmKw57}TW?96xAHulzv1^{sCHbSD8l^t`_eH -z;g&uvAhSqe43@0J^0EcI1(Ij>ClHYyghzT{J|!11L`7JB5i=*M0Y65FcVKCP@+4`B -znk-i*aJQsPn+`Nmo3LYk=a>z|Cqz!v!ld$~dHBcMMj%!^{*btie(`5$bcLn&O~n$^ -z*An|#hUVBt2UoZY9XvV;;fN_rjVgWEnj#55PR`g!l7*3;y!%q&C^AoL#W+%{vJx95 -zmBy=d*Hli=O{e4*2#{pG?HqgcWwoT-MV70M`IF?$^M^+APHy?8ce?-iMnBUxO54dt -zty`^<3$|k;S}C<8&4s}P6_GRX+m-;{vA9{iDeT+(iD|Pe=N@@dtXQc4T3P*95<8+# -zU^lRh<=DX#foY>vcHtIeVB7UI>3H@}6mJ~sM+zgQg{8k29+!l7kmoiwe@9_8N8ixg -zlUCGWp;GLUUOJ${Zt7cgH-Xbu(=F3;F4HVq0@IVFlAKXrCr*Afk{5lc=lm@Hli&C` -zuPa5zA6W;jrtfX0K^HbC@R)8o8%gU`bpz^cM7+J18qZJn>jiOdd4!mlNI=GsWft17 -zKtZ~VHeD@XX}TvBP7E-qtynnSKC}!nWCcYKE`7|4iAK%zjBO?%jZ>QnT+Ra3IlpmV -z2k<6e`tJM;_PC?OnRCe}9y#-Fl5?%5O)jilV9K&35Xu5ldR2YSuNR8O_=@(@MMY3_ -zz)B;UOwxp3cA1`o($X_(Pl4*X-^s59q7ziBTlvpEp7^i!aj1u~1DzmOC#T~zll@R2 -z>n#dHc-dM?H(=G7Zs~M4o4aTPKt7KVuj27Y_&)cA{eHACMS7&Sq{8iEr4?4dK(e}$ -zo}Kf661+z -z?lA1x{qJaQK9-d%{m}XVj7dSKgRJ~!yFSu$Mr}%QL5_oYa(@8t*Ev5vJCl9#7;)(* -z*AI`Z4y);$&E#-l4gqPp*$Sd^2}I?5qmZ|EPUExFN1ks)ASa0ekQd+wCNdrIg|Hz8 -zP^5I8&GgmAc|0e~v(5k~Ks82+Pml9fT&Vw-g -zI_JA*XRyzMFq{CUBzZSmcUw)HeR7j-b1*?}B= -zPi~&KJ@m_a(%b9MTR!iWv|IV?(A4NrdcXS*VgvpC=hP4l^*6k?F$8-`AZmF;&+ZYC -zo~cvtgUNgpNOyMT2yD`mzWam&!%Qhw=%>kai*=cjn3)UTm>x}Bx%Yb`X)gmJ^YvWmDvcmaNEt79w&aUlLsLG*~MVfmQTfVWzGy -zFUk7sexbC$nk5ZK6 -z&!iR1R=oHZOyaXmezzwgPSKo!jLZR5&Vc@z0dw|%GhcAzYss{YTMEYWbZ=T$XKzdA -z<@o6gR^5b+L&}|=KN;A^fy{`4apHK@jDam`wLkX8^nu^kk7-p4lMEy${vOw~GR}6~^8wkbI19L0lYoV3NY%Rpvdam73osMe#L3au{&CH4j55oGQtV=K_Gl+5R -zTHDoYtF| -ze=uD@u6PFj1_}LmT)p~nb=fNq`l1*+uC-NO1FWC$J^mSb%J5TwAKISRTvLyKFY52t -zV*ODqy$ulA9@OH++?%wD2Dt>yedPpuNIOFkZKnHV8Y6t^v}ICLC#aHK_ -zd6je?*lNByN0Sjdo8CnNUEvy>e9w=JDwKMq^y-kW&zqiM5;OAmM1m5q7BCj2Jh2pq -zv^CTrrMDz`xANpolCK_{$+-431GI5rwMqhZbg|FQrA^z(O$n@>DMS2ZV2ha&xm}X% -z{aB8Dd>l()MS4CDy2Fc|HYL&%9-Eh9^_9aaewy>9oQZ{}$ib>DSeN3B^c+~*?@P%% -zK|x~vcZ|`BaR-+E3~6hajEO0kCmuoanr9dj3(9<(W17c&lI?09?K&@An?Y`5$i}mF -zpR|4;m{z`=xjtxx-}2}Uqt>Ua5GNn!Z?%e&+cIWbB~#U@x>}O%~2%NFfN;N -z8!8;wbln%@WsO#hKd{M}Om_j$D+A{UU?!2QRlX^JT><9%F2>oemeH_Yl -zB7*e7@8dZ^x`LqX3@qx{)dt+Si%dESzwQNz#y3NbF;z#dp5xZ}Bd%g?VgoK-4{LSU8r=z(8HYqyr`wiC0WUbHww;M0%jv -zvQ%+C$B;SOUC5cqb*lsCar1TW*w1W~HHdiKvX@2TFC@Hsvdd9oS+M%dL)q*oEZjEkXGaa@=IpWQJ>@wKGoo_^C+6poNPD5+&y1Jme@Xkd!ZOJc*`FAid% -z62wzbxW4qNWe+AEo78|uyd*YFjo>1lcmb;cSA>rdZkdQ6>i&2va2f{1J0ek>B6E!X -zH6K|;08JJbcZIoM3Wr9;^TPq+za;n)6H)=ewF$B#{))E(EuWF5BUa%TY$*Mgu;xfi -z|H@T!;nlXns|dBW-ZTnc0Bobyp_aU(eK2L83A1_Nf4GqieC&Daw8#G9sLfkTsE5nLEDeG -zA^Kfg%L52o5L4MGfPfw0VA(ziGyVwM5N7`ow}R0N6-O|YEp%bm2;A91J_$4a2-`r-Kw(L -z<|r7saPgFI7%EQ13d9wUNo6rG6lRP}ePKHX_QyWD8A(thrNWt++nGP&qy>8Ary(mh*wV*t0!mN>qqxsdu^UV{}{iTg1E-0DJ^nd -zxA3_jVNj6br?BEM4lN*VPHMIvCd?FgZFhvf^3}HAgyno%gcSa%75PviT)bv1>fe!! -zXG^IjcRy(vnmc79G1d$!dpcD%jcQJ(Sm{)D8dWxxYEGqCFtesmWs@m-PNh)B*kJpa -z2f~>xt}}OqooCS;G+i+b_@_IsSOs$Gj^en}IX-y}SFD5tLlfW96fAym@=Pv_^mD04 -z-&ASz?rP!;HrxQD`$0^Ua26Sey_i -z8_XWHxhlbCK^$k17D_06BL -zGl${+h9V8zA_(fvg(GOV_`tALSpC!7_tU_)t;+E|K -zg#|jV94{VED4w0XcaGbW)bzLh%D?p=LUqVgZ>>oWY^vxleYLe-ZPD#M13oF6_|3o} -z{agf71%$;JF^|jJBCO(6yd4|orv-I$+P^q@QSsy#6kjD3hs|%m(M03-Q&AjDu!mD% -z`peF+6w?C%i>pcVxQK~H7p?7A^e0KrSWg~c%w)@cB#!xk4bDI#E;8JIjnqxuD0`Zv -zU)67187O@IVHpqWh$#-@ix*3Z!?@zi;Nma>v>FZi(Hwdh6wlFGs2+rD-dIp$7yLlg -zM|03LNSHge`5pzbO9sl0KSW1BljM%g{vA?4-g2SVP`@%Ttw>rs0d1UG!gdz~`yx%5 -z{U=KT%G1rc{nm1csh(dqVRJgW#NQUmwVul;E)970DzELC{?fZOA2Jg_s`~KKe#s!& -zk2l}dF_*h%n?Sc<~kTuxGY|NQj+j<=22h*!C^uC7WIO9Ey0 -z<|8E6QQ6rH#Xd>P5q)xh1P{%1AC}M^<)V1Rc_Xo37&b$jffXh6gFcYQZiFuiH$-dd -zqLy7HUbxCYJQUh89T8~ -zVI%|pmC|dAZiZ*M{zSKKzTIb;Sb11)5Crq;A+8-)$zI{Fshs?ID6XluKO4v3?LmYt -z>-jm)WD;0xmLPUPfT1aPtN9$^z}*%sn~$jlT*OiSJ)tbtJc1DnS!3&s!| -z5r(ELOH-Z$FF65f-BuLg{sIG6ab{G+Ioi|sDP4LuX3=>-)aU9?_BjY~mBitvKhcNo -zBc#|NsW_;&^_fk5r~-{Jek=i=45HN1%dT3w=cvOT6`mZe^W5V*ukrQ!`^=mBB<~WE -zLZY>GkG&`qoU_1PAw_!Ssla2Gxli&c;dJmNzY?yZu)IM!x9cB$kOhg-USG8Caa3}J -zjzX4Vy9BkAHhM*=;2n@u5)w6GEf=&_3!L==6UEn2qDxMruc1i2wr|y44aHYd -zqdq^`wWqBKiXSm||}C`osV@9_{$ -zO3(^hqDInlUQpU$ZREq!SbDVd9qS<9Ge|<*D@u0B)11wMuJ`(&7z~k3GzHqQb`sVf -zh*k#-8fk42ta>;!c=o8!rnmZF4TP%DQM)|ZB2P*n0Kn34ZM0IVi}65Ls`<3jISf9u -z3@uTDURX3jd`BR=6ppws*_vO)isy`Q-xA0cg(D7~uo9Gt_sk}ak`xjJeJ>MWtWxY@7pYAhDVD!bM5<}5wh3wwMbHx}tTiB0dDX8Lbb$--nWr6idFrFPp$rNYw%swzLB2c7Q -zA<^IMg(-yQ8dpeQX*?%rj-nukhd8i??WFZL1e6=x+~SkSi?h0{*RvwAF&_N0R3 -zia!Y!e=SgaVN3HlU6kz=8>+NUp@YAn(=Um{%r3t0wMcPj9iDfF+_j*Nz{Ih -zFFq2YZ-#j31IXU$5JTzXPoifXibElKYwt0pFV}u9L~$_05R)zL#q~#@6tbTSc7GkB -zKm5;lg!`)yw8i{&?>Ryp^Q<=JnQG?dAot05v!&Otbj~>fy)A~wK8WN(WT&Dz-mo;s -z5cOxpfe@5i947@_4133_{@$yN5s1cDYBX7;6*EOkk%7w0Y{Pcteb|HcofkPZ-Zbfd)NhY{$jN=LZO`R^p)^b4ipqydO{;tDQ7^exeYR%ayV*II^#&n)J^qE#Kx8QUky>_@ -z)F}yJ-=qW{tCgw^)_p|?zeq^nG`CPqOM7b&&T?W%lzp1A$3)wU777;!pt1U*-maH> -zThC^w3VT~G0VAV9QGsZA90FHN42YJi=*6}ddaafWP#T~dov)0bxfOAh`!*W??hl-*qZZzHrdRpZ0>dPoq3C7H{90|)0!Y`su2(82tLN9d -zq6#4Ddffn0WIuP*Ui1?5$z9^#l-6sBdRZ7psH0vqv@w)FPgZKOv~ymx&spe-lF?OG -zeFopk)Lw7ANx3;@b-7?j??a+@l_}z#-eYWU)JtfL>rd}Ol4X?+zG|^jU9<=S_E17x!gw(iQsVLm}1#xhYI5|igDPvndCbUH_EW1v;aGilhf^jk( -zg?n~w6W)y$o)LMYUJ$-6LNArQ4~EaUT^V#!F;lehI@8|>;e`mz4}`EZ0+hi-)>;f9 -zpl>O9W!$a_P|Oxf&tpsMHpE%BideQ1)_q}r6f_tCMasY^LTH4&=oN3&qS9a~#m|g- -z#p)!a${0_%09LosK3H$))d+x4D8>8yORpKg%(m;cTQ%dhK`M0hk4Chybc$W9K>IU6)k4b2C#9df9V=w32}#b_Vo;izD#>I3_fXs -zMshUeBw0=`J?bS&IC>vVjVqZ4F+;RcQ%{pKK(f6|Ks{SB9p#seg#C)*75}|+4iUfb -zaG1jjX=_Kci#^1aD9f%GgmN4Hn9 -zdsl!yEZYhjnxU~I?^|X2+)y0`QX7+K`Z@y^@~JCwVaUU?H&IXsMUr1wbq(8^*HI^5 -z8+_r_YbZAxdYR1Xe?(V{NdsXSy^?(p2)_i*6!uDX4Fq$Cq!DphC6vZR2*-3lvsWp^ -zd0`dt0xc`Diw=amI?7x0N;Vwb(fg#|*Km^Fydn|l#)AV7){`gupb3lvPe+K?2tYOI -z=UnkEt72->a>jC20=IBkMl&pMIAR@mK0^G4Kv?gu&$wdFxPp$lA540Y@uJ^5+`JbF -zx(pln9au6_yjmc<#HwOcaWuP$Y)Iyi9ztUx3^A#t1NNA-)lR}QD6l&C)_Ojy>mW@{ -zu~$kj7n%rZqrl@!rtP|=X^`2Ja3#+~7&xXmdSv`!VkJ&R;TBKX^iFnyOB -zg=j_Th{>zgq-}IZCg^XvQRE?S(@nRK=Z844;fndAub|2C?AVEGTskGnZu7chr@tPX -z``4}x_l+BF_1XKvG+yyaQWz9s5C#4|ZM0S%m`wn1@N3=0Z|%3rz8sGxRC`OoEiBte -zc||ileN{I_Jp*&h-TX3w)L*GYDIzJ)n$W}a3ibQTjrsJ-H|@( -zhk8gOE;Nz8FOU8`xzga=MmG0S7roHW -zo^@UlR7=W#OSf%&3|(21!ZmD)t{YSFBdRd22lC1^!Y -zt*b6kpLi%+e+B3a@il~<0hoHd1A!41 -ztj;tW)qpYtTqQf{9>8Ff?6Lr35vHTtEb_oIC3Oc!A=25~fFI8tLU!37`UgB1M6(R? -zc&$8vwf=v}I?ZTbdm8SfTL$vlxNSQ=B3d_+n`z7yvqD(QsTD(jRNH-=*h_ZWe4N-y -zcER|faLul5Fb1rbDY|;eHW#Fgb^f}|1!)WasH^qW)t2m3`|7Gob~VqZs_-2vf;6>+ -z?97kAH42yJ(%mS_{iBxGSIb+n)8(t>D%oWN>XFuS+iXxSdkrWFZdH{ivaCo0_hinra|$p-mca8N6_leMC`YSBCmIECQy -znS5T&SN{2O0WENeAs;LmaR?v%*Cz=HA+Q2GV(;c6?y2NsxYbH1#$pw5rLIU%rftUj -zSa*?_Zrm=u)%qqe_0?b(@t}B2fMU-mRg_yfEn1nPX=Y$z!e96K#+lrzV8iKqpjm~R -zxy7M)a}*XkVWH*pL*ZXX#OBN_UUu_p9}ch0MLLJaj`h03SZWb -z*YuC@ibUW6k;C3JsM$%Obgi5Zr+PM#$RY|tVJE*}phX5bpLL1qVz@8hXe1bCU6}iB -zR_`YQJ=HVgEQ%K!CaFcD`f{~ggwm^nvwwh5L2t|{klhE=l3*kk^z=M+->8V`jP!l?24=ZEF=Df%>uKA -zYy6pA1yVWf;0FIDdyloAG{~o&+~D6<+0*I^SAO3E=o?n3b`xIFbO?G^s9mBN5OA&V -zd6D8R6JXO!fb~&209$HUNPOj$<+$=gKcW9k7D+ -zezkW$-h6`6rC6Jf56M|^;3(K&$o8rRY?ssgJ4`ei47R&) -zmIEg+a0TrKm8!>*lypUXPLDczqw!FDEw0lA%SsB17WW|b{IZ@5c~7!!M%&Xp7mJ{n -zAx3in_S;##k(oA;psq4Df(M^cR$V2cZtqSsUP`H(FTRvw7YV>wduTq|Vlv%yFNuJ% -z#SuvV*B<1CmlPu!vDszJ)U}g -zHUm>-_bBN!5V5adZo4g9)_F{y*<-%njas1lC6hKpm)SA%te#pNZJ>t{ua0j19*c-S -z9bMLf)dr%OH6P;HF&MPnfV*P#{Q|{kMwI|$BQ|V)Y+PjVw3QM5cLLGGCU+vx}m_9d>zC@jGEqX1uqH-%CtX+VrtE0;*u_mhf5RR_G -z>^Y3rMZk1oCYX{$c&MAB~t?Re$%A4IM0|IW` -z=sr2pzX72Oo1@F##dgePqx%-<(_gW+>)l<~yP-yJOpoYF^5hXi_Nlg}Qa -zA;?p2AqZ%S3Ha9zv?!>>l<=K?xadnH~OL$KODV-i#I=y -zu}t6xsKw8*b#B6WZrkbT=I7vWN%0B0;t~pbyc!t4p=5K!r!p^SYw^-X+MAA_{P2y4 -zd3}acmhi!T!};Bu4VF80uUt^}6sE`?RhI+;qgvCg9|j5LQ+x;~Q3)3; -z5uEdNqu}Pp8G>xlo$9dBFoa+4w=_#I^i)-Txd_dL9kcR>jb(SJQ%M&rPYO2Qq{kmL -zi#-PP2>R4Hz;|BV1_m_f7+5|a&%RV++{i~^hm!B$Ln;PND(Ptzu1I9Sq~f6sTDicO -zz<|?+l@~-d91VGilr7O>QG|F$%9&)ONXODZCQhbN55u0k5$Pp;nv7s7i%TcF -z?_nL&zzmJZoiW$JMFA1h+|Pw{#AR=Xv34Jt1PbCA01i%wb7?M_Y~g$Jj{aAAjzc<%5GCO{mshI?s7b0tZ&J_ -z8(roF8&bUo^UVTtP&ZJr$Etj`1_W*UF58P$wRl@VH*B`*@TL*n2hX6YjjO}iE;e1)607Px!&iOe^6HcxI40f#V8iHZ_r2HTc4@%nbA~)kTbdS{sy#aK -z;H&$dB5iND$F0w)oaXu9FDR`We~`2eT# -z*ZuwY#ptrrR8=wxgM-P~bOwyj)kL7~lgv0ZjLK~A4K~6JC`M1MLXG{{Z%Uc -zDwnQ~4g+!^c7<$X*+j7+c`>&<83kh@Qn;&sj8Me!Ym>!uxbgL3GmWxd=Q6};AW+AE -zPZgtBj2mZhBj)C4H)Nt$4_PiopWHS2#u-;14qldi+(?YlwPwxub8Cw=yNa4Izyf#f -zV1IP>ZpvMhlD!S?gsZSOUUQ+ppkg!`Gzr{xM+(EkcTp>9@xjJoW}@V -z1Dqo~-Z4XRJkPL9h5oEx;Bqr2wN<%J;qP$I&&A6YVCZGta~m_|qfszRh_x+s-FVKG -zk>~2lrkT->tFE4}`4C)G2<(LkCexlPD{)pX_W57&vPGCd9&B|G!Uh^A*SMXgL552n -z4Po!NT^`S#LP`LTKuY#B3=xXiY~fFS!g{|u__+59>XKw^Kldg+xn1YnB-kvdj;c74 -zykfLLn_nFjUT?_Hce5qJIa%%sw(#oHD7-4_l%Qf46tHOX!y7I6`L#*mS-As{VAtmN -zXTWlrpU+}o&YOa}8EF~2x9@&iwG4Xr0m9i2Mzqwp+b2~lirXe%F*-LT?z8-o(UBgD -zC|Jk~<82jQ7#BP78N=BJ!nc!SUz!m+J>4J@7OzZwFiroWi+0@4M^%JeUOz -zo$`a@@#dx2=ezLi_b}hligw&it(;a#@^QQ5^2rdejnk{)D655ExQPHir?Q1qf8mR& -zh(ceqP)SHz_<2KF`@W@1ZY6Y&qnGJu1z{=|6fPm4Zs`R$m!s2$%+zIiXu*88OPipR -zi&6}t$%TI>bOu85jlDSPwWfKgSix#y_W(=tq;Lzebpiox4f;{9IT|z -zIK`FTuF`JJF01S=Usm4-jTpw!J_}0%T(Ds1ZcNs>C%f{W_vSCK$zvsrd^ivU%|#b` -z)dCZLJX@`OUKT=VwF^Ez%=FdXsngw5)(Z^sScoQ%+a)org&Gi*)DrG1!IC<{{o6=M -zJz-20O1>bB&S1&j$qF$W*eg?0pKpB^#pv~2=8`V-zR|hvB@Q{H5Z;iyh}m{a-Df3{B(pDZU9XBL1WsZ6~%ZO -z;mnfM%41k*lraQOK*hQit8AbJsK~Ci0A0Ph3w(_*AMEq7*$lK-T1Yrp3QMBCw97o9 -z3muitCmenXbE1BImpQNtwMa7w2ca<0^A0AwOEk6g5FA{Bs*w!xOD{XG&;sbWHKco{ -zHiuD&m;LJ_AET3?HA8|AhzW -zrtssJH^-lNBM3`Gr(^}mZB(&gW3E$x=z6gGm>vpr?{>(ADIuu3vW~aC+QKwrg1!u2y -zGB|4ss|C^)!D_(2o&?V@?E!|Jj7~p{##96XLvfmrZrxtK?BURqOnQa -zo#bJiYvO3hh(FIpSr6V1p8y7D}QYjZocT;u>nHc2P0p^3 -zPDB1$XaaG=s{s~ZOy|2U0d?&A-h&LQozTD)!sD<~(bp*GblBL^M~rkB<`0ZL>mX%( -zG?KnrI!SnyCH7|vx3ExXanLAUzwEVOtv26#`l0MkLbxhcYT=+ksJS}y}N8r}T0WzSRM#s4A!K&UI#sk82}*J4^2`)g`Z -z(v2ji7v?w|d3+1&JE@i@mA2EdZwS!NnokH&NJGgMQF2~62!rozM4nV>hcN>@lbe3H -z4BH2&2i^9KSu#jE-Rt8N@3?I4fGl(X04knHUn|~6W5xQ4l6^KH+b{s&9i^K0QFefI -z-Zh9dQn0RrC&5Bx*HgS}vUlNvvf>EE^7`~k6+U`Jp#ITi^B0%l_Qwl}6>$je=~dzE -z6(jU_F2g#3_PPraWm_Kk?mK5-O}j2bhqdu~`nyVR{)&!TCW01EP1ZTtC4QQALN+tt -zAOx-uvM)!RgupFAHg*K!45VvCQ+ikUnf^}5W(#ja;8Q|&cogEa)J-!eZd3ofr80qT -zY??v*J5?1Ztmm>>|eKu9@Q@{aWZp&KA>f4UYWgH9JE -zSwxczB7jQ*m*!w0H@OI@mM|p|QWbqa$&^B`2Cahv6C8X=2@R+jc)Xh>&@>99jijUz -zG&NZhPn@hL93w%@E3KDJF&QT1WH{Z`&-LW)<1ew##BU-Zr -z$}CdnGY4U3VckXOs(~rE222j|#UA9|z6ytC1R4)%&@t%V$vAdRJkI)s^6?6)N%sk1 -zJx1&Z7k8k2_mF~yxs -zB#QK$jJrmQ0jxiI)n$twLlwp;i58s_?t+%;OD`KICnCHe1zvWKr=LX8>l9qqc{(I|qm^Dc?VNE% -zt6XyW>_{7}*2bbAFeJ<}zhO?&szh|e2m#*aKWDGiMu -z;zS*-0-~mWSY6R-yL^^i{uIWM!Xw%$+*wX`1z%PwE0xNdYWYkjjZE2j>XQ+6`AjGY -zeiV6nw%s@X!8|_)5-fg)1>UxHe?9xMxwcarlc@K*Y;Nl`Mknft%Vy#-LZ{z8z2$U? -z;mr7hJAXqv&Ddq5<^g)!Om&Ji4-5v(4PTv7p?Tnwhayb%dndU@r(9<$9}NNyd~Rl} -zAD8z!2BF#Xa-12A!877zlGQO}VHRH!vk}s&=nT<9EYJ5w3JAft6!mnH`MSaaQ~FNzTssot>pO6a31xwd%ch4z0P6*%x>!A9iyS|67_Pz`jY>M_sgY^B-URy -zR^;}Mz@0pJ?=*g@*5(zz4mS#+hFZMZI~JUmoga)0QTkXkpY5y)1G@(@tHYE}*rEB+ -zTRYWpuMc%e8@E~=f?49;*x@e)0}l#Y!ZBOiUo0hSEB4ys)|}`08RFh@B20C#)9IsI -zd7$%72waDOqIW}_KDvOKu^muTc>lN;Fn@bzQA6h&qRuqYT%v&~3wy4Ck!v~F)ISkKXF!pq(1h$p$)e3aA8HSb0 -z>`{z1ZD*IZv*y^dHM1rh63iHRa3rs?ptD>=dp|^n1nX8F)H6Hb9yB9Szqu1Mgcwh3 -z?xV~fbk;QTn&3O58s1pIL;be*JJ0X@kfp}DHg(o~lYL`jXU$V0{ld;FL3uJ*_mXM` -zToBk%g_m8Vnx5}GB(lP;O_co$)%JGh`7pffC#p~2bP~e9-Bjgw-VxEhG*R>SROOqU -znw?fz&TqBJhXhhEuNy(Tg7In<-n(%~1d0=Ad`C5{>}*>Jtl+GPSk25aXk=eQD@De7b`sV82r}d0gcC_n|;Py)H1yQ{3XcM^8 -z?!oRPo0PPF!gnppDtzT~mB_FM^GOKnZOS~MvuT1)yz)W3VlO232?}AS^D7}8WH=pG -z;j10xq8k;CLVEoBz*^$>%ga_#m64t1$j+L>@6}8nt$3d&0&f!Cbfz*lBtF1Uy}Wu&TjfS$^am8MrX~pc`B^4CJpK9 -zpAcbXQF#E(*;b#NoZ*|=Y4i4cgnyq%3Z4(Z<;IH+xEUq|`wzyHvI?nlmCpmfK6v4E -zi21sM#g1wT%^@0eFU;&D08Qysnb!@xS(dnhzqo;=<0CT -zU;aLO40Yp@qvl(j6+x*kI+RHho&i`E5M;M -z5v}(H*jTBlZ(D1N&dwk#@$X>{y>nc#Aqr}O5;#I8k^97lP+rT`G;Z_bCDh1o-E -z1pW3}{E*UX9f-a@X0CK-<5Y0n3&PqG6(~G)KF57&qR*b$YH-(4wV6qq9ToQw(Nyj@ -z#5}mgKRjz|%E@iNf#I83o?m)T_k~2cRYG64&D@+X9vioqYaAQrAkJ}*`JqMb55mfW -z8gTd%IL3+uAI&s4(N~;HM)! -zrLBo55SShpL3h=+&(A-mrG3O~oaPRzZecsC4X!V9Il;H^g!6;Tl=&V7k54vI_Q8iV41 -zd3^3wacLNB*bTL*pF7mtjs)}Uq2^{g#5)oe)6Myb`JqqT1W!A_|D#EuTNg(_>&Bn? -z`k6ivMI+pQOAwC$PK+bs;P0LI(YzdE&b?>+i;tI~c8uW>ySKmrQe(2=@!PKvC_uqa -zqEDU0qa*b34*Hdf;_nF^CHe`DxjeMW__u_XdVQ=To`?SI{xG3pC*8KO6{ihVx^u>$ -zh{yJ+d6WaO(r7zH{98sLYMrRZsN0bp*@$>A6ImVA7ni*K}(?rBH@kbHHlAsHj -zc@F-?%(BZ=@rLbi#I}qx)Lkpx-NSrV4<*l>4W|2aV{~X=CTx29GhwNv{W&njKM2e# -z7L7#dD9)nO9cI+Qp+Omwe!RtS?@O+jqyM%;95zzn;-6x`N!n?KyO)1Kpu4o~qQKb4 -zuYDi -z=F~fQWo<{{GHiVMi=i#aa65yE%_+#sTc9a0l^1Mgzo=NqEc{oy4A^15@ -z=2ljBl!w-)hnMr}>(Z6V1)D>k5Fe7s`IQ}2Q)c@WUkolERV5BD53jBk)5S))zw(Fq -z8D3X6CGUhOebt0NAMy)tG?^A0h6iC_|H0F{2Ao`mx$jeYWk>BCg=-Y~uY3T@WZo@O -zdJ}AFi@)N%M9fcGxvoRCuA`2lRGQh#C$!uYzneIdyGHks@ts5oT#GvRJ8Z~F?C2Kr -z{&sN6z1HD=IbkLj2GVS3pGla&!-Yeu#+y`2&+Tq@#K6gq*u6^#p#MGg1OjKWAdQ)Q -z>#=V%>E({h^V7y3 -z?xQ)1*8BPlTKkoDfC?r}w7Ub&6NfBm(8WL*&f}84YKJ2E(T^5J<&B?QVN*@%P!{kP -zM$w5&9>oUEiWN;inu+Mk*xDr&y&OQu5c}ydyl#GjpH09uYC-u5bOzFRixQq-!wu5V -zp|UWa=&dXq(J=fi(GLzY8d&aR(4V{O*9frKz=e=wi+t8s%^&+*psOJjH>i_} -z!@{Nai-Kv!ZLy!6&VXM_>^}|{3yXh>kz*YNB27eJI6eR>mRfm_rRm$Cm8GRw2R>e -zI&eKCmr1wURkzxw29DXHK5!@$!7#Tht@O6<)c$^$K!Lv7O=KKwx=}FPOQ7lZwHs6Z -zgCSoO!auB@*hMsOZhiqGy6$BRm9`0&4EcmItb6r(&=72Zu8Vo -zn*m~3oW8kTobPK8@+#!?pFg4d8#R)ymMPLmbFzxF1|TdFDU*)2lTk4h>exV5tZ2wD -zL6cUZ5Xn43l1dD@Oe%hh4GPUa{s_e+>!`fR8);aW60H@SFyp}qgMwHOeAwvQ&F3$< -z^PaSA9-t2b^?TaEhcIMi|C6sKw=YlE-u7vG*Qf2ac-u8<3jb>5>OsXU;g1jLHo{cjvqT$k)wzrkGt4};YV-Y%(D|N<5 -z2I(?zPvtRTYfPc<=t_|lXEynGS=rupdyQnD -zD?!hure)ec(|q5!6roWe^d_3t4tf$y%B72V;T0BG7X?#9Ze=NT%6zqq%*@;VzjMw) -zyzlSr_xpU5Ju~ynGtWHp%*->-%sivWOI}-(kk+2agl$hAoo4KRPR-3&tV -z%tS~VMprvod=Fy$tY7D}sr&$E`>~VxO8C_b9~9(Vr%gmi>^9%kWPfps3D5kv2%7|5 -z$t0$BUwOk -ztRWLYYMBkF{#msKMxIktq}=iWg8R|xlUbeiXkrsP8Uk_Q{j%9+=q!WpBQm8^{Ly^z -zj}B+xnX4Vu$LKcXX&{os3Nl5Jo>$>mKy@Rw>gmqLk|P1QXbDC*qb4A+S~W)!8EZ8} -zFn|EQr5;xQgRGuVAD09UA1 -z{W0eiYRs0CHocfRDPT^H>NFOflk`HFi}BiY1ms@8U7$|`NG=3I&qU&na3_|xYG-Q8 -zhBK?k?GC=(Y|H5=OJcqvH#>wjbJ<*Hc0;;w)>WJ_H8p=)W8C}<#&`R3`IGRK%BVJ+^YVbWq;@@i9H-`6B_&B-Gq0@uy~22aTAEhy*FZnNFD@F^2N? -zIht@JK<2HT@;hKcwMyo#ykX_c0>~OEuRM}iowV7>L$p5jk0&4p|8wm8WO>d(^V8EK -z>`8N(Qnymecrm)Xprkb&y5(N;gTVd7AqC{Q4%^NS=8EVv+ZJ||RWNtR>JHXvE-PWe -z9{bjJ{kLjo1*5HgH;0LM>RZig-+FKRRs~6~!VTX1(YS^6QBIkG(clX0-~%5Ql_jny -zWNTdWQ_C`_C=bb^BIw_FW^M-@JYbisVQoV;58@GuFCAKk)5S$3yK1)qjM7WtID#Ef2&+GBH&@Pw2fZ0 -z_O?<~_E(#;6N4Igz^!#$`C%!Q^L^eHXw%;G+dHzpo6qg@fm -zsIA9xgi=?Pce$E>-&Gk1s>`+FTaeKF1sFzO#z(>mRet!krT(;-?>d-8z~(Qu#O8BM -z=;lJ3PY2~$O$WxpH4bb#OI@WSHvNbcKd%E~Rl#Li&r!*$@$Ep@#79CMolAt)SYe$u -z;U^Ui5r&`cEc#(!>JLl9eo#TI_#LQl;^e;W~i-J;)@2GR<> -zDL307@p!Cwe?Vz=W4O4%m2g;pDaR`=oMl9r -zq+f?Ezny91-!@k&Ia4?{b^!No2vhl5M8EwzxR(aQSqHOF>>36^Ji4GT$iHA$l4S!c -z#R?aP!DR~4Ul%V7lh&eqTR)uA^zwmOp~go}o56kqD7QYmtRogwNpovddI!BhCw*r< -z*>RvSa7SLY&mc>2AbQstZ^LevKCYSxm-S-`NNG)@tL;c(vzZBHlvb)9O8E&oDm_HO -z=9x2hMDL$pQlx1vQrU~t6-7ST9UccN=T$rBoo=7kJWpkxr>>aiW6hhFJx>EYBVFF> -zvQ`V9LPF3lm@^M`L_Q-Nik*eC8f(dQ=5wY$H&Mea_2zO^T#*OkwUaORtny%=?N{zu -z8OTKPN=5$UwU3Nk`^a?DB&L7Fh^LB9QA}(lMHnGeO5;G1f(!8ys`^=`k2$&)v~tNK -zxr~Wi_FPM0?1AW|x;=-E&_WE5CD#42D -zZp=XUDtN!>tL-}tS!ZcK>t&-+G3A75ieq*NF4H^D|04tnQUQDa;`GO%k+nL+!Q)cpRyY1a}-XeXpw28c#Y;TXmSxd%GoP*5#7A<)j -ztmT)PNm08EpF5fH{KIBa*v`DpE;W;_?K8DPhM5$gQfru*%wm3)nHj@oQDRo%j}Yj? -zuZt62h#MCt1XJR_WyDQ!{NH<|NlA|2Z)lh?LMB7jx7!Z1@e9r5rFQJE -z@Qci3LwlrFTx=#UqG%q!>p_=@_;nzGrazB(5gFD|dKa2427j`)9ha;aCZ3&SR;v`` -z896wfooI#$#m118K*>I8X6I|!1zPcIjm0pSETZTiHIs!vD?VZ-3wmlMK4f+TDl9>R -z$vj&5d1jJIE1zm6b9+F9@n*HJg3LfVSU6A^XC~7C!>qz2JJw95xZ%dLW6c=0o&kT@ -z7OABehPU;Od(@uAm&S4WKrKWhHsFI@5B#V -zPjhZ(h}KO5A7>_a+HCi=QD!MSrdV4JP74gJlJ4>hYJ734FNDVogWQ@Kn(QD-I -zqD(QyOv>D4@*z~*vl;S{yG%CNOx|y^Mb)Njc^s^Jz1On91XA#DdF2(s~e!_e^9K5DQ@7?4IVm3%7>C(+rvC -zMl9Shkx6YfBdzK{;kJn+wb?ed^8HXg`dOKVJlJM?kd``2{KJ%6;v(ZvB=2V?W8Fpa -zH%(-8Pm#j!CNdmF@?cY;^`Y!VCU4q23bU9? -z`)@Wjk)W13T%t=OC~)9Y8rMcCu{&r9*IJ@|$hB5mY^(5xi6~?O`d5M-#b*Z6J -zDGsCeiAvGi`M+y~jjH@D!1*#pDt3nkeLR98zqUd=e=})|bRA4d?JeC7EyY-Ujs+uC -z_QH=ZHKT4f<-zci7mR$Emo6=J^A#P`RD(TeW%vxzk0m;c!$ -zzMv8RALTEUOx(M(L_^W#pNys65H)-%gFo{abjq4m+tzd9B@=AIr(NWqt+qGMv6oC( -zFJL?_R%SQc(NzYOU~g)Hc}V -z^;?;kizZUiYCC>T=rWP_TWxF32^UO4p$6ZW -zr@t|gUG(>YiOtjSXxMOe9L^dE6MV@I#A7HND4sBpZSuDtd%`5Vsu6QE#mX+wJje&9 -zMRmuhlxc-7Eu0OGf;C(?qY-$Bx`wQF^HFe^$kVO1rzJj$^`>H_Wko3ArBt8KH0Q_)_cmoPKG=PObWabXJNy -z(@NJ8Vjb?%TTfqyUbn1O{HKYmXtgao#~kHrd*dywe3gmPK*4KzqvyAZM@*o@Hiv>& -z_r@=7x_Mc5t+-Q*2yQHy)%t-OLDaVrd$>;QlX17hV}-bGv3 -zZK1e6Tp~@TbnYM8YEz!W06mcZ%p_9nM5bkwa;kw}Fbo}fu0s4oyYmrlSupo}0DL3p -zruuM`9w$+)!lR&~INK5yDF(zY*6@L`cvM4k-g8m7SP%-u-;@2^1(&yMKO_qMdJ^N< -zw?HR9G?s2Um#c(TuF^z{v{3LAe@5}#L@gKX!v%YBOO=)palAGbUNp6_aL7kdPv>7p -zZlQ?+M#JZ85Vy1+xpz*S2~xU}x25FTk4#iTC3D5o26?qFjMNsDOj&oR*bOnEozlp)0UYMb} -zut;@bv+BY-LoNua3kN1$xTX5Teh7yvw|8$YnV&cb8hALr!Dj_c< -zVVf%9169IjszWeeZC|@Ndw$`xiVIg&T^RuhQdurBCVwrL -z;Y$vm6UX>T`!|*6u!-{@e-6?(`iJJ9`_Z|QkH1;oT-^1dAuoB&nvqMEOu|{O%uMMZ -z?sly4^m*OZ^#PfgLty@t9mEu;k~dob1P5Wbud&J(XFW14l3jFtKwd^+xvRhecopDr -zsJQ`SIm1M%t%FBV&zv(T-QUHAog26n{kaW6 -z+_VU8bSO99i#&BsjF#9fZXG6DFtBC7nlwyUyv9nm4`@3jhH}aM_ba)KLEPF1E_EO& -zXc6>&%vwkulZ>ga*dKDf9oz4WA@{aReAho-lPleI!G6VLjpVpCMRO&XE^B6&78AcKT9o@!n5!Ti#+TXM)x>$LYPfUmBvnSf>bX1kj4@e(0G=FO4*J= -znO%nAwXw?*%#h%774%#?iqDXSTuLuSx*uGO3HF3Y;&)E4m|E@_T-bbW -z#kj(gw<1HCAZde>o3@0Doy&!#a--*QE5gXPErN#xtu4E;(m(1#i$2zBFuaWR?#BN_ -zxG(l8&StvcDM{KM-B+2-Bx{cxpa<5Oqu3}))t*0XrU6VYOdRN2CQ7}Joris+F*#1eu{8$A3 -zDFqm&iJxI22U~1MS}XHdS$H%Qz7nQMNo(YgIMqZxZjrX*{8)VNZ^7!2 -zIz45lm`GVmh4mghdin7wCh}p6Z7K3VKqk{x1z$zG7IRKOEObE!wR;@bGPgvhmh5b? -zZ3jvGR1@2vCEHtUi$Ej!;IIEbuGe!kVM6vK&zk!!HSYV>u$Yp0(D>yY2~_U6K^cl41cx- -zUd7~=cH29tT1DE=3S)pk2+`~fVo1wb8p)3~34t2AQYEL)3ZszBhf2v!XKAGHfJqF} -zm~q~UIL`|A)7)WFa{XBvDcolgbQ->&rm<5D(KK!m2WT3{h~b+2I64)$&tervbE6;S -zrls!e;8sL%8$$WF>G<)su}ayF&VA|E66;fJ2=!@G{3}&TozzahBS2TC61D=6E$j;@ -z1csLW%BKaDdWtuFp&I;JfgEE)s4g0!Yw#7`q&eQ|n}hccg?SoFx(Y2cmIGMF3py|w -z%lU8xPSAOZ9~vi2F0f9{NpA5-t}~ab#E{sp;@B-ls2f|1lDeTPu{Mgnp%eC4acPis -zG9*wIFo8AM`nMd|mu=c7Iv6zgWh2Zlbw=oq1cmV>_#=LbKkWP3+%qn5VJvm<=*g?0 -zt(1DQ-q0bKu*Uls#=0}-N{5(DN6ehQv`BJ@0kP_Tx=N!f -zJ^4U7KsCE`DmsSD;IY9)e_q57bParuWkA~B@oEUPpL-9ENQni0N3-aE^B01$W>FWp3Sq9K@9_|DSscYX4w -z15CK>gEL_J=W)UFvSBB>q_K?9vwt9sz3RjN?gN9|eX+VuZE8j^Mh&HcA0C@F(x=1_ -z{6P3%F~c}Z!!9+RM{74}x(vZ3)-yua0BG{i_h<%msba(aY1{?_2M21?Mv*yZ=`_OM -zjT4{tB@fGiJ8|r0UxymLiq6``HDf(-J5F5TOGBd&5^lw@ulYjAuTWAh`!I1#Y+*() -zP6DD`!*cJq`0ssK(MSBw2ge!V*D4rLak2R;WqMn&r5KEpfcQ(_e51Tvbe7TDt#v$W -z6z-=IrxhN+kMNigmR?TFzULBtGm0DVCWzcFTw*Yn4D~UNOVwiq_fq~oVIqr)GyFSPx-uu=Q1PYrBK|PwVO5n11Hz{K)~#3(S3tY6gj7&n{wctSi(QX?*RI#l&znp{QWrcB`+#5v)*; -zRygpqMwTM?Mf&hY${L0GDTU)PMeZntIz^E(S&=(Q;h3mUPgJBND^lVWxju?%N(L$u -zyTUlFd0N-BZn~)>@q_y+fP6s+kEgq|I3~OVdrsnTuyRFUaz+dr -zhU33*E|&i&C7ORCmirw#av}hJDAkB)z}zYhaafwPsCm(spIJFGOZqS4sVEy2Je{>hJ0nV*9VGS!eNU -zj_U7J;=)AQQKnq2PE?fA`afcv);+D6i9=ra%qi>At?=uDRbaMXH>QIp{VW?iR~Y8Q -z#Sj+k57W|_{unWt@G3a%+l=LAp9T4f}JJh3t#M8u%HMdCZn6L -zWX7QNW*Jv>q5UN03mfIk(~WR=DSVH5@0`x(`SP!!sl?YL4zbgv!e~=rjFbd)ic$Rv -zCZw`%hCW;}Sw@jUi%ZFN*`xYbDyxrFr~T{aaxcd}UFBYjRr3vtHMuxUG@& -zT-*w7ObQ3>EWKG+$RLSPJrEuy{zRwDpD|^^fgX;YdzM&N3GWO@%rHoAXNI5yTEaBq -zY$MGOoPs%dBAne>!fM2aeJ$Z5OAY)L%$&k8UuI&8pF)`K%S7W58}S$t;e$zld-8Cx -z%2!YiV@8x@L<33gMGQpItQX8^>f&ao;iGa8tNu#biyqr-6HimMkNqf4nCS~fw+B|i -zX2w41%YTPug7BU3Dn^SY4dZ9t$mS0aX25tRPJ&J$%{8`GWG1F&VQ<`cO@1fu8miw? -z%>+GAi5C?ytl+;io_Qo1LfA3Jk_?;>#ey{ljg9bD8C6e*lXZn%e|zlS== -zBlot#@w+1Tx+3?QLMSYSz(JUl03{E18Pb(r# -zM>~rXQSj02%#OVS@8il7rwkmnexRATa(rM6S2g6|z^KpT2Z>)A%YNa%!aA+;mlEr0 -z;iUnQCiqZc{D2afx$?Lv17p??9K!tc!@$8@Rph~eVV}o`L9ab#%s-4ut76Y<;D1Jj -zz7elX%`i2-Q?9IN_NX{nAe5o8<0=d$(QNECggh|VAYP{I5L)O=cg|R)q6QTA__J<6 -ziIo?rz~E4^&}_s55@NHlA}X|9c?t4#Gr2Ui%%eDCdtlz$%~7SnZ3 -z^_NOuS=jVVL3_@GD!9E3xKNeTZm&KWzzm{iJ&xv7q=&I5K&Eg4WHxE=D1do^uwnoN -z>1!hxGfUGJPTv)}cn^UIRn=)@bvh1nI{N{(?=XHcvANR~;y)F56h-@m?#j&OFYcv{ -z|DNT2w3l@c&Q9AR0!k~qk2*{H1Ly+dCh5~E;0>;86l2r7h4iok{-yv7dT5(5``OY)w+$}@c%@1 -zok0H%U>}l@=qM7fc{#gyK$Wtj*`wsd0r5#=Wp~=^Uomib8KPd}YBuQh`(5g8d@rxL -z@$troB6o;e!`MUS{1d3TSFAKUEadMQOKR}U+DJYCqZi+au`9&79T)CKm!8p`7>%yX -z7}KVYhD+t|@O%XY6#W3m-Y}*ePOR)^u2!79bD%9}mLL41Kv=&3o9%{>DUlikF2{hO -zQ+D;glxP_Qw*%O{_QkLudSY-Eeu$B2<`)}T56_bHNWlM0;qeN}lwA|}yULRE;EJe7 -zdsIX_F2LWbhzibMNLwr>QYB#>Xpf4tL=NbRim(g{ys5MdX85b1fbXP8B8N=aRn3GJ -zOwGY0nH!G+*~>A^r`mmKhldn5(aeA6qLU~Ywl@e%FtBmP*-VC+L$5CyA^1+5SKG3ZD%O+ -ziuzmErJ&~Y!Cwt!UP=j3b4xU?N;rq@A+k_$H;C37i6h8u%S=y8nwB+dnnZ>jGV9Tydlo+!M7|t`f -z@JE=)jMVK6MdRlWOw9-{u`h?-^SnZPIqix5HKjT+&Ih#%b&)rP+B&43F*FkhHqXMr!hvhxJ2-=-tFDYcW}#ddJ1iEwe4P)L06F;zH?}5hC$qH&ff>RzC!C^N%aMO -z#71M|^zNweZc2;rrVBfep=HdL;_Q-)5bgG~`K4gcT;G)U@1b7s%Jb!QHkXbcx$aha -zPGkqn@`6`?TGl)FfV;e7@F-{OfpZ$^ua^&A*vnhsbH-=|~r~#!0 -z@tE97N{kxtPm)cj0+1@gNUod@MUg4KD0`drIB@zy=v -z^zHMzm=i#rn{iM3Y1+bKb_v>N8?^r?jE==13d&TxD`nw5XLH)38<_9zUHhmgrA^Y| -zvmQJBT5vyc^Uy-W09dbfRB!tPsz&d;s$kOtvu2+juX!!Jzwp}7!rhlD3}JvP%El$$ -zy@%YO&`X3xfmxGISM0uY;Ba>4j_Pt{L1hjrzqX2bJYBI``C{8ZVpvx1rnw%WB;zXK7Y2NCU`~CQzd7 -zwl33G7FmxKVPjOQV7lPbLL6i)_k#8xYKFi5p`rQx3z5fDWrx+}RPY!af>cA2Q<_+ovujS^WuJ=pFaOK4Swa -zBz~rZ&wc3K;mkC~gy&Jpy~F@>WKaqW708p3vey-9P=DZJ?2bLaBR*=<#oguCd8&`g -z@^$Qs*Mj}Ur5IBFUA!POQ-g5r)pts$R!kpSmN*SpV9_u*oi~N)e`C*3Tg4Z>auBJ= -zwJ1{36)BG>95d-^&M}^@?@~r69Gt=tu22t9kkT*s54G?`>p3MnO4wsutziXf3Q@xp -z5>#S6z<;kJ)-UMRZhH`aHQAh~P-hC-E+rD>{$9OM#&=Op` -z<}*JCyF2hU2YuQHxTDpVT*6o%OL)|ZApSj+A-+dd*oS;@bx3_x(~g^oUx?r6AU_WQ -zJmo8w2t8v*+Mz&xluz1%K$7)^&2Wk;!=M2C9q+^Jt6a$p3Bph9G9TdM{`~g7`d5O`|lGxcFbZE#YG^;QZ`8DnraS -z{!PQbVXX3|+ju3`Ch?RG=DqfKewZ2FhDN9utrds3;|e|~Dhq-ONYt|se;fra2|rFK -z2RYtw2@m_|4*v{t=lH-i>&}KCN1ltH?W5a)IQC&5W()I1N01}O#Xju=bLUKICXt_u -z(;d12it(~hIAWPG%!5A$;o3ZKzUaa@G*$U{^m0$(H6!z)b!v|8I~5nQm>cp^A@(}| -z@GYskql*iysJpFuJRPT}!B{cpew#uRE-%sK29Is3D3k -z_)IOVY(Ki36D^Vb+viyZ2H@_RTKHH|rA9x>P^3gExciir$xmB!tCM~^rS7M=G|@7s -z|IK-p!2uLRc#@vIDb+G6N1Kbg&#SQhPTGp!_#z)DN|&!Q&%=X=?b4~FA5YmGx2k94 -zh#AT=g-xeh0<5!ggte#!yN*@^djD0`qE6vcUAap5DxMjcTj#QH^+{iya!_lnu*C=K -zacp_w1h0{KB&ACFW*8Pwas8NO>O(GJx55(VkMcfuK>}xA1wEg*@J{K+SJ+jwcdWwR -zQM2_YTubSPRi01eg3TU6xsFl6-BBU5j?tG<+qnwMeK;m9e0i6dl(NTtPx85#Q>i7H -z6aHV$`U>^>Po%3k^2CI(WA10C{-&HZz)+I8VBUQ12NJ!fB=+3D!}>c*!IpTz7+QkM -zfa9@cgz5FR3oftdvR_k~uBp;vO1RehX$>(q!JJPO -zBu#r@jr!EZcTdg8zFb)5Tb0I -zb4lh5py-a--4B`vo~;yqaWTOT4NN}>*h@AgG*abnD6FYqMDMlIV85=CEsbODOc?vrjxH*7J!TT`|LA9T6 -z5WKDD!7efiumGl07?@^-ee|@mv=$$}km8As9pPS^llgQk^TXX$AANfcTN+Cq -zJw@%Hu;=cLC8?*Vf6((v;RBpUAhSNcLGUPI1ox}AWk~;V -zKCcx8RMg#Ld?WT8D9f3p5r=y-p40k|MnxzMHF%jT*0#%Y{@fL$((_P~E~DS@k*A(( -zYD{|I)b@*6?>3!Wfcp^5f!eMg(>tDP7|tj&AMK*fSzfOZW*cvk%Hca7Ha@5JP*mEY -zsv9GYdtGLL1oY~tlE?{{U -zj0wxP=6WcW*8pt!nTW;3d|2+iLK*dHBRngqZ2xL1S64>xV~lQ=j5j)?$ln{eaX!pI*kl-- -z;Hc8rJ>Wm_rb_Wf;OQTS{5YJta#6tzX_xi&;}7Cpr-q~s<=%+>c+1nXNeN+OAHWtx -zlV|(zt2h@_npE(8ks#uR_n$gsmSXhqop8vY|KtFpa)!UYZ~DxGJ8sartC~v`Q_7XO -z_bD#cHC=izf3^_nt=oZ9KM=?SBgXFlItzt)-yA+IN%y3;?MPE>;It&hdxHn#Jt}aP -z_wwLTo=@$aspaxK9ydkgZH}6L*6yc>*~66h0Rk=-_w7$nD=rOhk75S=P=$N$rEw)1 -zg&eO?lrvRce5jXcl1*rez}5BV=bN4yaNMsXj<8C;r84N6Q7 -z<3eJ&A(6>Je)b>+ryuN3zeCSJJiVl -z5x{Ryl?3`!*awxQVwF@F2qFXUm}hosAdLKEYLg|{+#J-88Pu&D#Hg-ao1}H&G9<7x -zuPm|$Q4bRKm2X!B1>&P(hiSrs}Ki`oobP -z1D_>+*%DWim7%!(EZquBP7Y1BW+o?xv5)q$GhO~a)`Os>9>3$}ZR59lhPOVb|adnj?dmf>_Ssc|i-zxG4jEuV^$ -zxONpz4^40KlMYRXaP#_Gmg1d`Kxn+nR3e^q3ui4Vntf@xaz+JvFJ_je68Im8@T~G2 -zZ8-Rz((<&r9G4kXl?LtG^w3N9$y+$NaE-%31WOpSUolMu#qCiQoZnDYmA9z)r&Pl3 -z_{wBB=^1z8;NcH2Ie8dy5Bvw%o!N44@hQ4MzAV}O#al5@Bp#&R^F`8~{p}TZRx|S)pP@j#nC_YXwuPD^#im{-nV9s|6Fw^vuEuQW -z0{z`F#m!Sz_%fbqTmqfw*y}2|sM5XnBPxRGV_gq%^Bs&$|PQlfoqj_y|$O2x<67InufIhfVz| -zGpf%`K5$@iTLcu0+?QRTkV2V5z4Nq@OjDDR_zP1#nMfs`FyRdHC2A3Dc -zOoM(eey0@}^at^~#cI$WU#g1K!bwKkQk9vhHt69RW2WC}gPsls7^ld?jcmYwj$_js -zc@rJ!Bt4-qeQM)lIq^@u5(n*IBFi)s><-}e4NRC2!i^m8+_{r$q65}@dcT&V-In9i -z_{jR8zItL7w4f?adrd%dO-NTwsCXrYd7KHEwsdnad?0!Xb5vy$3$2^>j)orpx3Y=T -z&QANbTxmOW;?S=ZO9LvF239N$s;FHg#=+*~OlqHdDZSz>tmy~%cw^;UM7?BR8sbc< -z9NWIBd1-Kax^vv6zjZC`-#%VUHwt15G{5g+u+in0mvl27i_p1|3obo)^P#*^nYb{G -zlW9M}H!)Mr-i47J>k@W8V;O_xmzq<+$5A5Vxh>ShU!+2N0iwuMT}r%}+@wKPMgupe -zH(`N~T)_X)e{|~T45mN&>SDY$6d;UIl^6nru_`7M=ZQ3h4YzP_LcCwad^t67TJph9 -zho8A|azb4}XQP#oY%7cc4{_vQQ6b7&!f$Q)pcVpY<7Y_KsHOrYOXg -z7>rYAW2!veD*(R^&~6^jU+vG-VaJo6$UYihwidQzck-b_88;Rxb3b(n^Jx4e -zb%hJ;LjT4X5REZFo>R#iCj}2l=Cc$po{hw-4W+^D^!$O>)S}erz`K5{{ctPJ`cctq -zRU6Baw!V7u^c(KCh9VW%rAv8R;lQ&(uPIV86^>^WDXSIg429zfMG9VSnWIo+2YiA; -zjV2zhNQqLY2Pqu=6e*0=6#sf-w0B8#@QpFJSBa)G -z8`B<1NMjSyZVhA7)W;O7{UT%6tjNJ!O8un|a@I1wmPmE`Qp-S(UH)CsA&kZ{NS(LN -z66yiZDq*_mFz!QeU@@VCHmhxm4SjW{zvGZEm`k -zR*pS$;+9of`&7jp`QJ(H7rjM|!Cj3{0TGxHzOrM1YtB^s>qgLHsZhfoq0nAd1o -zFs1EBS*vnVDqPymw5IU9b=thNoBG0NYNOtucQd$$y;o_Ty)I_+YQL0&F71i5Md9GC -zuO(icw-n6S)H?I4CY=+_}eyswp(@;aK4b)$)wSnRXeL;MbCa$j+!UB8fa`~ow2 -z4%#crr5kWTDvb&SCdD!jbfd1At6t~5n=pSuLb_s@6l^3TI?zblLl&j~*f7u68_!5xJ4aWkP?$uT| -zB*iK>cY7a5wu3^prp$5azB}6s34#9uV$Zhrf#h2$4(F{zMc=p=Bsjt4p6gNSb8^fb%xLnQEahD-Qtx7@5zC$N~<ek04==CFeacbrK`Tw5Z`vgl*diLa( -z+SF6#&)05V+(-^vL@(XMs`59ME6rM-+NZ#vXPtsCz9bp^JXwiZ=E84*r*i`n3aG~I -zB#h}&*0nu>2jJLyUqC@)V5An0GJ!wsXR7zosDSf<%$f -z`rNs_k=$?YnTE{dzy*^oOrqUf;`4#8oy>lb7&9Z=7_fo_yMXQ=KxCQ3kO|ZG?+J -zcVR;l*Ga%VWq&A@Ejs#^y|;n(38B7k-0lW?Zj!I-8~45x$JX|Zdsm7RYus_9SeB3H -z{-=Cwl~Y#s(yl)}lDr{B3M+bOL>)IvQT%c@Tsqz!OM0~dY^+$813x5Q&*e1G)sxc$ -z&pK6)YzW-slbIW^9Je*}*|-Q|2}sBcp}Mez_LGcdS;fjVuLcy%smpwnMx-vw#$Oys -z9N4@pll9dg5HSMwjAfaYiJz5v%rHdkEAy}%COz_MF?b~{Z@@m_@Z}Bo&`&I)3wL>3 -zlw^6C%uk!U{t(+G4R-6x7_vT+I(84=7s|2x#{fhet@~p#L+1C?UFvVC4eS&D* -z?%uH|zk&;5IYR=M7)T|TnqA4T2L2I=G$|C{Ak6zS6xSf6N+A;4K$FF}?n;0T?~G8Q -zZ?Hu}%9_tA#py;e5HRdwqr*c%!tp6CHVyzxSc45A3L{K45*0Zd?n -zEl`HxQ;fu)COj^|Xz|If7VUV=!!v{MI`|aMmygJj@1#2tLR4=Yorw4v&q&|>;oJLf -z&2@6YAzVlxH)JR`$%lJo=%#ME|FJa-oS+{7c^M$XxDYcpWEeLontLU1yBGDtxOJlw -zJ9k?*IPrF5)&?g&O6>Qn#YMRa#n$!i+*@C8I(qD%(gb+ObAQ~{wa!SN&A6Hv%nb=R -z`YAUO_dLhBi(Tc;X!a#5ohW;y(-Q1$F~siD!21L@BxZjdH*z!=GTCAX*rkE7f9p~w -zHr%#mII(3YlO>aFG2Gatp+~Y8JGU-&R@mCVgFcHseACH!GZ$D}vMT*Y}X+hj?|#03yC^hpW6%(D?n0F?aOxNr%Oak3mPz`B{y -zE(#a+0$_ax~K6fe7n(qZgN -zM;wkeWy*B0QXzcmpLDR~Kj?^*K!mw&I@tFq9lE}B0H1jEf1`uXiebix#8;x{T}qFt -zFFiQHEUcI5QOfj8i{by~?tfV}+J9-EFZ{pzJgmHQ{D*vBJ1$*jhiSvj3p(#yIZlT# -zSPtEB(wuPVIMqq`$?o#N`ETWz}_`W$#y;lf7Nm<=xd@04rmb{+zLYq_sd!-kS -zw8({Ci%|qkU_uo}qK^yE%TI|=JXss$`fa+7^i-pb(al*iQehzzQWDm1dRYzrA3Cs2 -zu4mTD9^TmgZ{E1`NOeANoa#92{q8#3K040qB>RuE-^*2|bdi0>sm66)$$t|=O5NW$ -zb4WgLe`Bc$zU-Dl?D;?IYnAGIUMYSVvq`%(iBgyydnVI5Qh!1ZofMi&HJ*Gy=U -zoCDvza|Fb(zvD!Zh-bF7@$%iAr2Kl2#poEX#39-Gw4C(Zdi|q;OGX6D89}R@xGYgv(`U$FRHgnUDP~YMDFh3Ckk(Vdplle@5Q8-CG -zanuV9?q-zy4?joCE{xua%fNxkh`j8%N+VUx4NJF5d>)=b(+$x?iX-}=RLcXdq(wA7 -zePO+Rjw*2o`FDD$mWVx+;S1`qcheW4JXY?+lkCE8N)j!J0uJaLiGepF*a5F&0Vc=! -zv4?SPk-ZBQOU^?=r3A4mCVzo&OGyV)s((rKBIBew#BDX~JzbDLQg6)O|MzR(Ka=j{E>tu3(Z-FWvt;6oVryz6r6|L_uAmj!Jb5y3m8~o`d-p6UiQoxv|nW -z>W)Jnhbvd~x)w%WT6eU=ZxVz+l~$QPo8H0fC*2P5lRx`Nh4l`!!>bOV;?KA!f2k}( -zrk=h*B^GNJDV_)7DqmJR6w<768X$_CI4Ow`NUBH`x;Jv3AKa^cE2*QN`g-bxxUCX**`nTd9CKvSmaVIZ?omdc)@}rBalZhyZVIq(Nu6-Q*TQN8u -zO;*b}-t5V-(jmSPgQJba;-G^s`$i1&I&zF7O?7;JOm3@N7vX~XafWa%c`}zehfBo4 -z4D=No6(Ns1#Lbe_SVs8U)`EJny-v)NVx*yAjzfG+4gootfoC -zQb_K9y84;8Hp{*egGUR3>&>_$%jd-4(Sm?_Nu0A|@}-y5qz(&t5*wfmh>SwlskS^a -zgsA1*nK6-C=rS|O>N@^qFmEws-cX0|_n2Hig=NwZa;J`B7XF4x18y8yS|@J8H{iyR -zAr5v^OfJ?g*!#z6CtT+7g1>NV(&BD;=$zuB?aDIP(k&x@ApfePH1Qk2Q(%cCsdd7O -z*xyiJ2lGH1)c{D4gJ18io)kx!=kH)&=o^PlNqimR^L^teoR>pb*C!4#rjQsWuIiOB -z#EJEie*JWB91W5S_2RRA@=~UfggW7wK5@Ny#ix4n+OE_xjfAUXSGfUE55RpX^=+Q- -zg(KH%ZP#mYA96)Z<6-hkt#tEWSl+jFBp%p8(W6L9J^R$3_?Wkt*-ShWx#?eczQKf%!+hZ7*2H&ZWTr6ZVaI$;TP{FooTMKqj+^pY4W~ZUK+y -z5~p#=k8-I?xQtcY+QF9WaFSI|#q7MNFawa}@AWpWo@qSqDNd)sjlhEEn-QEwgX`;U -z`g$fG!KpO3t~YfG4L<9}#7d#3>utdjCN>3$=RL_&^|nA{W+m-a;>pfLcrvjV!bwKG -zO(WsuCsIUD^pKP+=UgC{BP1b6>rLIzM{@?*=t23KS`Vyr8n)f6!v+4?_3R`mC=pMl -z*RvC4t|M=6nJM))bU)OT9Y@Q=~XfU)lIE)7S_XPQ&2!i`Tas(fbAmyPX597GRsa*0rF7*j6W2IXje5KI<@(@Sy -zsOxPv?UFpi(x6gOSwOFmhFo<$NXx@p=HOnfW-D(ts(ulKXt|IqCZv}DgB?RApT -zm5rf4ir@)~Rx;;m!iCEL*7N!aU+0k1b>bje=obX?Ud|`hDx2KK|qHw!qe7e7ad$&&PN8#R< -z@rC>qTz(zz2RNzk{iRH1)yb~aJOvHK`S;$SFAcub6BK=Da6@m=+o1WVU{wb5QIK%T)Ci5hjY>;G9<4x&q|#I;{6>q7lOe@F{t_~(PVAwgK^uTHp1A^l}Yw)-z3)phK@D5RIW^q>C{@=h&(i9+74mFg^B`Af)OYekVlUXdZ$ -z%YO;^Z7usPASD?kmjI6GhyMZK_+{RJOZ_bY)MwnqTI`w#7XT^EgXDZI9Sitxq^z{< -zFa>r+absC?LBe0 -z`cE}ohD-G2lGV)3*Mi^!VCOA)20C2Je#up7ny{D4KJP-@d8%7}TFbZemcxW`seW9B -zhFj~+WvjRf+??C_hn)MvTDGkx_nO|^AJht`X>QQ--^(6D_xn-TRW9Z>%40}zE&D@n -zd8pUhwRFmp8b+@Bf1_cvj73=9gC(z4>~fb29Ir??go}OgX1e+1kRoM%ZF2Lgz9;T~ -zrCCVtDQ$JF_+?+*E9CMzd*l8Wn)$^&SQgg`Z8V2W%X|q3-`a}<^?0<_Hok_r!yl0% -z0k<=Q3mVQD23r!t$?RI&XnFEFgk&HXG=c{a#3(Ycb{wWb@pFXa -z@*b+SalkEnh8S7z8Nme)wnT@M(eAuT@e_*bJ{hr+r6CYCT*Aaw^yEWSJbDDwI`KeH -zT>h_?;4osWWll0UbI*S)=Nuwo5m+NEB@?vclgO_dlisxe}mvtz{IclGe?KY1c`anflQ -zK9GrSs>YJE)h-B9P?{1+vt8ID1<7f`vJ3D18He_M*KT_kjU{gFq1(2Nw%t2+_MLx&E3`{e$iLMC -zCk4qHcA@Cs;EH58F~0}yzxKoZaa^)q8qH)8NvBMn`7b}F(<$3bIrJ;j?6r_)Q&n -zP8M@ZLPri+Y!^2I1m#L)7`ViNmgI2qxZU>%BAnr4f`|Y-)$S%CZ*$z`QiIia4y>y4wWSB&kxPhbO5pz!ox-DvAbd&g^JOPs- -zc7DB_Z-JUb*`>uM{{kodl7V*N|4E4?!fvyo7RUUih(OWfw#Vt;P>t4@1dc8(`l5$+i -z4>|dzhR$o?V^7?Fkx{%i0~qo{jga0WpSSM0=gEsT;)3412)a+b|7`acB)!5!T -z#@rDf?i07C#`gL#$j0nGaqrhigAPCI{}cD_0aa9MAMge?s4ylVp$WpUVaEFj(8^mK -zAhS~I2x+BVjuJzfbyB;XJe068HMIilqGrX~Fd#Q!LCRf$O0B5WZkBm6Ys$*X%)FL- -zzh}+944w17=e&P>-}leVtlxT`XFcm#_ciO{!PJaFYUb_g(#x}*Nvyn+KMoFbu&2YH -zm2`^C2%+L_DLpTBqVy=&6VWDQXMLwJl}HLr5(q^R9Fudfu0@ou -zgZXSN`RG$f88aR$J4F?%T}=>SW{sQIsbA~M8?Zr6r|7?>2r^+)n&*0?Q@O@X=9ae< -zsfIjom6uM$H_1K5y|v`^&P7uPNn`r-3 -zb!oEdG}|jGX18=2lZ6iaLuw6g>NF+^r&+H5(rHW-PIiMSUNM~_u2jyo5B$`O5o%_t -zP%%1kZXmn5lfMR0+LbErB$0pa)Xfm6&5iud&L#=HwMk|=!9TVVB_4$I8kM>Ua{okw)MrG{J;)eHa5&- -zAEZ-gC;x5-bXR>?uX(Nhom}3DqaG7f5C?biLp#s_#1pZ|54~vpJNd_urNAursdd(8y2u-k?eh -z&7`U9pe7fsS%^tfF5FtMnrL5 -zu_laurQ#>*oPX}ZP2z8b)pe7zYtCn$JhaXun|>s)H62=CAx*eeb!dJ5C_BGHS}SDl -z?Jaxn|0)|lLf8Wxy1yx#T#Gw&zdy>pw?mpPWFPJ=`|!V!eaK%_`|~>VKq1D=H|C)2 -zwhqnik$uSD#BS+O0(_X@&FC;p6--{f3)L{Yu|v0c=-6c)eB=&H`u?u4VTS&z!Xibb@SlamlS1NYk$+Hr63oo8lw*vqtaOnko&KeTF5E1SN4#`wfJue_=wB$TKZ>FDi(-tqkUeo63x>An -zcj2`D&lT3UgA6O(p#M{0K^-*hF1=(NQkcDi+dHtH>m}uof-Mm2_njuwW6Ea=JHJyf -zJ*6E~*x8-L6g@`&RAHxg^3yw|b{;ODjP -zpf2Lce%Og(X&m#!M~q;;-D!NPh#c>+&sRG|fA#}!9AU@9JE@==`@LBDp`Ebwl8ha4 -z%D9|$|1|A6ciI|1t2YG4?b(r3*dF9>Gmfh%YdThq -z4R+z`mX`y|oj>6BTc!5*K0)T5VVo<+Y?EKqR|YoTGc@nDorSOMTopd)h468#YiD^; -z2zj;|E3i>}=<&q+Z8+Bj2v)%KC|FuRp68W(czSMdCNe^+^gO|0yPT1H<%K}GZpykA -zwsmK5m8_o;yM*@@x5riO+)WaJph+Q?4~ofsZlNZGUZUp4yJJtLe6*ekZDz -zi*?MQbif&InQN2l>P{*rpoI2=pvQhNve$~hr-J3WtiR~ -zQI-aLk=}kH9lN!$74S&)ax8zwu;oZ?ZuBzxJSex@Kk>sc|IyRxaegHo_m6l$uKsx7 -zy6^Z+JLTfix=L9&EQ0j>2POZUz;$o&8E$`=C!KPkS#GbT9F66V6(VDr*wv!YC|d+X -zblAQXZ$smE6^QX$TrX20rp^;i%4aBl#Seb5VMkYON5>)Ddx=|;XQ7zx{Y*y5!uEn4 -z?cQ$r%`ih%T*VL7hxsciSs(nJHih+UzQg5H8!L1Em(RWedbduEMf+a9bND -zQI;ymV-cnmMZDAJla!2$BQnzMX?}fE#-(D7rhIk2SV<2%OBc`Y*9MBzTX$rt1^7ry -zD4e&z-sh`+QlIWqkNVFEK*e;0q;!`(>L2u5&m8|d`e2a|y|8E%{Ak-Qv{BoyXl`4H -zwQ<+%P=ZIYYj)(`Z-Y$7T$fxR3hdm{T;H93?5Z7ne=EA(*$VQ&7@EtTxWmwt=(WYg -z587}NflJ19zamEu6 -zD=Fdl`R+`+EYC6*)|t9Hc2K84o1-v!$BYntkylRKHzv<P|!(DwcoWcCq4cTh#(K10R0NeDz(ske}o(`#Fk6Y@OeH?Md3_11}THcB(x -z&Msf-4ZH9V>Ej%c>#E&}^EKsSdoc_omMgMfMRjw?-?=c?n4x5c`)grm%zO)*{nT#N -zXPQSJDI)3Hb_yjvchHf+(i$U4x#FHctlkRpvtd%ry}~P3mfdUdjNf{sD3L^m+A$wJ -zRbijGL)EWQ*r)CMhE|M|{EaCT0OBk9M*G{9>%CB>%L8RoF(} -zC(Dlxv^U!W_LdMOK`Nh)q#YB?cD3tM6mRQorrWl5ep@S+MqTNR&&5g0$U$^)2Ac*}d3|bM^{p*y5$+w^aB! -zK>Dm8z3W#}65+E=trR`P_A<%_&$o(&QkMx;$bU1|F^tDa+S$pmY$KUJO6s3@Fkcx9 -z3iiAONG4#t4=@r5c*X);DuhC&nXlKhE8~3m8eu=tEPbF2cDjZC -z)`H=wzcG>&ezcvRZovZ2k;DYq{3KlL!FKIWJ|2*L+rs@}V||!MwmEk*r_~tcquahT -zfR?EyY31_^y@;>||!s+qHOwzt&QLy~OPLc8KJQ+bZl>Br>HPBBk>c -zf$*5puEZ%7D43=`NvURzUnxE4S*D1_-<|I+Z>7IKN{k{6#!WBYduw?mlw{-D^=Kt` -zWV~rF#Ou;_UP{_u?9nmlh1NPp*vRvz@5C%P{9UrDT#;VEYFC`Qmi5YEr -zAE5(B*};;uHa*C6t|#Z`ZF-<^vd`N1(?ATnTVXR^2ihni%7p}vjS<)P+r<3gZ&vKx -zHk$j<&s(uK+xP@zcI})1NnQYZqm3saEtRtg5cFCb9|MPWh6v2?cD9Lyrpjp#k2&?$ -ziW>kD%*F|HT&8@|*ruFID3)nJPFNIS4Q<95ABG@mP5n<;^{O=WKW@czQ~#f=7;ED} -zq~4DSQ2doP?hl9lLjuavOKrTT8MZo>z+Pw@jVl0=0?>hNXya#r>&FxL>1J1k4H`Y$ -zrvH?{Pr`*Q@vNxL_%Xqifw@Q0Iqpug-CHY*Sa|YRK@0!B8S~L!du)JstPt_g4Xb|3 -z5j0k2er3h}(Z)XmukxjZ_u)27jgPF@_29(n1^q({?=`|E+6Pv;ls2*c`F$_Ou59Dan)-g?;bjxr_%j02 -z&48f`WoeewW_;jQ*R9`Zkyn&=z4F9l-N-g=zlDzHKp0colzqOqnr(=q&3Mm;u~=xV -z`|OLjMj>9yVSqgz-GB{38NfKv*k!@OMZZsf!>b26rKo@}YBS#S$p>%0HsPVXX5qEB -zO{V#xPA@N2zO1KJ*=30Xc+ZJiS=;HQ19qy_{9m%eBaf1c{oJZ`_2xa+N*;QvmG@|? -z*jA6H}T!82h4?OpoTWF#!je9J%1j8=tm{h1c7yGCwri|7~6@yC}GDUu(ho -zg0wlPqNSGdqP{ga`a|o)^(N7+jYWOi=?3}N=~fgMHdLRi+<16n7;5yXxQ2t$82oXa -zt#W*UqloyMRuR9^f>UI{-~0l(AWKv|&ag!J9Q#Srza7WUe~jo5p`33M7u%g`?(s*&MA~dQ=Rof4(8YK+r7X4| -zek9CvfjA>O*9>D_7yFy&&7nfsh!kXtOjmzpgF8KOmpabpFiHZGwAOU3RSLS!Px61v -zp7weKThq*+X~Iv7O$mpe3wtW~a7kG9U^;uPsUIF&f~R6yoZbQ%l|5?qsIfR>aMPxd -zBR2M#{>Zg2?AYR$G_%$9*6@j!A8w!w5H+?bq}e`l)buIk&mNP4_05519lp4kKZJ0! -zXJN}|c1O;u2f+rywo};>NV{dq~{61fqeo;jPA6i!_lZ -zl9blooj*#uI7*Y8uQPCVLZW@%Z0|lJS~yMlXo$NHt>-1ML7wuKOHJ@VQO1;9Z+QZC -zktk!v<6ew~v{JQq$X|bqc>1(bTW{oe{ZrGM5WWh3`_D(3KBwLL2&ilfKBUEW%8xYx&pouWFfgwRtkb2sk?XkB~3g -z!NL|}zMGB0)Gy#foB+DbRm(`|7M9XZW3E97v_+=baZeXt+0ym4?2NLFY8^IM+5O^4 -zN!mP_=e6vAJ8?tE+Sc+*B0qldSE(N@+wZsT*t$1qVpWR9{$lA89MDMvS1HpSKlV@y -ze;H9U<|M>D&{Di%9R88vz%FdIH1mA=y)Bt^wpD-E0+fv^trWD8pO -zmL{oMxjBJ9)hO2WWwjW8Nnlwmd_uFdPM?*)SDE;{7VSonB}6&8vxRR0pPrGxmm_Qx -z(YY;pdO~sT$-+4;sE7-2dnQeLQy8;a_`{%~`IBi0^4{6(FD-m6n6>M@@^IvdTyQdd -zdP`#NNv9T5q(#eD=cS~aOw)!687M!gja-8iWoiPus*!At-F4Rp0SE*BNaZMaPw~p+ -zBM=4~-=baTNl!sHu7&9IR4<*mwkfg2xTY7~s1~!m|Fb2GnN56UBl?e5TCnL&d;nQu -zvIUE7sV~LVliDN;Hn;^nJ}F0;=)qWoS<*0nJ3ay127jkXOEWGPweSHgm`c3DLyP$J -zYoXqOa=C!qboLhN7Z~Hbm{`x|*P={7WhqT(7n%_%@Sba?LaQZH_Z_`a@aRcC9(b`6 -z&BkSx{L6lB=HVy~`nUuX6*2Zb)~t_(<2c5?GuS_yjpT$1yC-doX5I(X+Gq;kNV6zi -z`Y0M7d;ka9YhxsWqrC80Gyi2flIhX}{_}Pr_dv7qrv(1vb}^>@s976hk^6pgJwspG -zP=&JKpMSiON~5|#T7~wNKC?GCYX_t(9@pV%X7gTbOk6xJ*Xvx@BrhJ9;&GZRIgc!! -zO%pQU_S(GJ3`>zcVTnG&wq`BCVi(x5u9-aap@NLkfd`1hCuSCUo4Q7x|nz}Yj=wNUT$%({+_!J1YPfj%_(>E_2>oc -zu4bOqjP)6PEZDqee*bpluOJI{d$TC-fdX=uhg;38X9RfYL>o7&S?Ozurzj&cn~fkV -zMrB}nvmR){_zl}J-G6PfW_N>#7xcTjnT)6Zj*g|-&H?+gz_j12*d$ZG^H%KgW-7zR -zSu2(-F#U`bi)*HViFVqGjWFe%vSNxmz9$6;{X~4@nw7KhzWHl`4Q)0~{}H>SSwCgP -zXxtakto>@i_^;cr+BT?JIq3#X8=Pw*BWgcbbqO-lQaR?)1$H^!B;xy>mG1W@Qb_;S -ziwXVCG#Niw!UgZiChA)$-+1XvdEhk+D|XD3_mvkD^1f*@j`o)KWs{g`JL0AL!CyMZ -zKX0Nl5R}UZHRJYnliC0ITo9VrM}k-Tr^J2dvSRNFU5tZXtlP=n0;c=~gzdqX -zVVcJs*6U6D0t{>%5DW;5bvE&>+oa#n!TdNLf$T5^eiV=K$z$l26JKhG=wpgK^||z) -zjc@BcKJk^WT9bXhS09KxwT)M9!~E(#Pnf(6PWJOQ<3nE>r@ES^rJ4Px33kfp2si1a -zrhn~kZTwfd)aU*cODgzRUg%P^o!4(ed%DM?OO1cwW!q76)K~R5N38BDwbeC+;?M+n -zEloW-7p~TNKnYzyH;U+Z&Nk3>KD6zRbW}V+$M2`OEp1Zv_~ItU{C{gQc3JXA>Tla1 -z?Ye}`;c9q<>WIV+os`?`IQL!2A9_56{>7wFe==nx83S@R*RwmD_>^t1BD=kb=Wf&9 -zi)VA2_=s)VUWCkUY2uT&DeuO!*-iY>Z3;YZYT}M<$~*8Rem0YGnf@L*$?ak%aYlFC -zbV^zL=yW>r^gF3O%?oODIP#=R@MlD(YrqBX%58iA<(igwcfZ=XO=*cgzA3DrNbVj% -z)pqtJ;`rL^5do3|t_cI!quZ#jtvAM7``Y(NU2mP0ZbM(YV-(`)?rS@Xdsc)nOn>Zp&34$ag|we78@D6$pK7;dr`e_ZFwVq=I_XiVfAJ^zY`zFUtBwatIJ;vT -z|N2#Q{i`Sw&)p_wL@NQRUI+e$(OL6MoLjEuq*`&``zOzA=f##%W%PkzR79fHkc3&*?&Cq3WC_NMnxAv -ziHvfzQ7iSizG>7-go~lw1+ngOD+O7bm#X2HsSZbW^3eUSonwa@&Hm$y2#egGy4{=H -z?vLH>jc)e`ZubU{Zr1tmw;DyC3iIL5!<+5)IJM`<`G(i|ES!jDrwM55J+jtyiTQB+ -z*T$W8E3w9)(F{$0K7J7fY^>hxd@dfN3$N?hc%&7pG5w77@l#LIzt=y^t(!xsl18IY -zn8I|v+-MY7oG&#R&zNT0WNN0EPLcQjD6fR3^NJdcb*2E}e5TP@YjLh?G}f5%*1D}` -zI)yx|-SSLtWDX}8D#znA@_IXSO}jnpj_Puc-PnVT)C6`bE8-<5MpVZgVYubTEQiRe -z_cihr+t488c<58_q5i2ep$<<+0;lh+x?~qk$RuZR**hjpo&Vp^&9}{;)5zyEo=Of* -zy}2%g> -zk!z{ifoF!=ne%N*uD#%q(A-C;3%xX3z}cOmE}C!4yOMOPA5H<_hp$*=gB3oB9rm1q -z$FX9eGL&D|h}A>D>W7nD*Mb!iO+6JYAXl9r8+Cc2k5K4sZ$9c`7S2LYT1~ec+i&SoT2s=)M>e -z*VU8J>+i`;DXo!oD9=@da$!bV+J`Kk3u*aJzKAv1s7Fi49*o -zND`^l^POPVb1qQ2w7i(wfnWj4?KPX^FP%v0<*_p6KtT=>5e#QjD>;2d7PB -z-$dZ(gKkiJuV>{TTlUpqF4i-n3Luuo#VGIgOby576@&Db)a+>w6FuY$AzF9OO0?et -zNyuamsJz{SN!#qZLIFzCM@!K+NEhMclBs?+cD7zSXqTe$t|LwBZ?(Hz0r+s;8dOvo -zh>xOM`<`IA_>j(>xHVYIv~?yCVcNZKVy&iY`Hb@(jo#?lW!`haY^lSEBqN~%PiI2s -zlVOf@O2>?>j`379le4Jeq52@kq@$A#r9o0qa>ASP-T6zg@FY8I@z_vIi3%?APR>Gj -zl!xcb+qt}*>c^qm|G%jpp{L!{Q&di;kcBiE2uJbq}qyHCOf -zsX_%Q_mNO}-mJyh@=~QRF=ZBt#(OmN`|8S>0oIDr*4 -zc4k5ekc=!B4r!$%D)H!5@qgEA^Z!{d^b=9D^Ov`&Mf0#t?iL*Bku0leL)8p->AV~D -zbp1tCx%ZP!UW?LsbX8G;S~QkUJWU&;E*+^lhuwm`amlKh5r22M?5@#i29~m`V-fCi -z#M`w94S`fdx6o(o!X5OYj=ygkjwqX%@UL}z;pBSoxGQ+jqTYUZn_0h~MbBK+Wrj%Qltec$$qFLkru~x7Y+R4 -zIR80`qIKl2x%)=b$AqEMz_>{IDsnjSbUJ7lSd{5Z6f4kh3}k>aGvD~8Cof?eOW4MT -z66t;iEo19)?360r3@Z{<%?U@x**JjFHy%m1cR4x_kYj%)RY+Z#ozh>5igSxnr4y+lDXpmm6ZLNv}o82Ij8#s!rmB4KqjejA~XMsenQ#IZTtF)aOWQD45E*&xjz;L -z(nrzU<3HQ!=nk8q88V-LBy7QK$AYtb9hIprc7`ZXw%?~n{x -zmYMIFEtM1Re{L607Mfn=+2GZ7dCE+x%jEs<+lo}%w4%hB<5DgQd!atTwbhn7i!ae% -z%`17e_@+=fafP{>c>*nXO}o_@hQFmPW&03hv5yOA5iH<@YAJJ05i1Uv!TC-G7iFLg*D~^nFH+mQX*8 -z79ZNzJu}3IVf46)=aHXgn0pNVkX}dYmR$L2-lX2?)fWMz-O@X~^2Rj8B_pK@lPRT~ -zb2fMYJ)<*wOIsWgg{hv}yMDdT-o<3Pp+-dh8VvnS` -zEqa}H)cqf-@XH1W%sEy+B2Lb=AKwy&1?+XJWo*0^=j*$QkGmHFf7-wwual&GpENiZ -z#wqI@>_7uwM!?4n+FF!YDnolcvJ(DxS#LM+MZ~%bvJ$ja4z{O(-%Y^l4f;w`7JIz` -zhk9>tu-6(aYmD_3jz~uzwzDA`(?!x_dcK3TyT1*mgS9r$G?Mg~g70Scx0dH%jqYzf -z*TG(G5Wf}s8_ON6-b0V?S~spe?qD?ySmajVVAXgl1&&?)Y^)sK`ZEqzMiAaQkv#2S -z#lRK3Um^de9LfO|7Z`nv?V&UjrfZ_->GRFL8|DL_X~6oHwGQ??syPe}`pXCN^+(+C -zozKO<4F5YH_SApusRl98g5@KRd9l?ET&csJw4ug7JQ(WE{05p3*Ov;&J;r>zf#&74 -zC06WFNTiiDi#@#NOwL0M+JnA+2~Fg&B@N00Zpba~z6O4w7WMPJ4op5RYT)PK&=-*) -zq&)@2?rt#d@6B_U&`Vip)#0zg(#G9Z?6wB64pPtd=!H1lV(PD1>8?Vqz_w+E@Y~>_-{PSY$-?9Y&BU -z9E&apH%#PYJ;SRrPbK~OHyAhflGnF^Pa^$hdgSp5q+dXTHmeuig?c`g^t-`BXXr8bcov_ff-Y* -z*e3$hul8a|AJwBCbKF!w4La!bt6oV1-MUH<#U1rZs^CIc!^Zk=4+zNV8hSjnS%f+(m@4U663x=XA8~lK -zzMf&mQy(vc0%N@1y}aQHZ#b;l411_@xfcVkZpMF{7ZbKCu2&|w)u#CJK@^vl>WwRW -zaY-r^+T66pZ3DsgT)lpog%9!zO!0r(_4XV@r*8N<~yjNY!Y?j#Sd(?$p~ -zu_z~`P9N@b1=m3)jj&jsI`TGD)8%iv6yZX6|3)^go?z!|^>{%eTqkP{htKs(tr2HR -zJ6=oPire+Q>5@IN^HRtz^Yd!chCeYci-r>^bM2W6o=BN)pA&Gszd1*V;=rz`H=+fT -z_aAezC?`s!*Y0My8&j_haT7v!*l)O5?(~5k`vHru=Nq?T{du?tOMP)`ZZht04NcvN -z5j-+vH4+N^Y=}GW=>xps@HJp%7uOpDec|wZl&3=L^>81?`h!<+9uVX9NQOKIDJW^bhj+|H7&2g@2$I6Ls+qTAJs{PGpTQxe;WcLm2-QVJJ2)ohZzoaLwME?n5pbM*a6Vcisl?K2AS_wBH0o -zaR$~cLRQXOu|1r}ZN;Cvvliah&DLE%W5MbqK}=B*IC%FkXp-k-|odanKICPX}vR&itxOWm1ox;^zq(sI-7sY?<5 -zvqG-^y;m-<0#3a$?K>;Bj*D%*%C`bEZMNFPjBhO13eLZ&K?Cu%1GGh^MgPK!B`q=IaLkvU_nG1~8mS|U$ -z_XTL$_bM~*8v88RM3Hy(y;k1KMHt$9RxDZ2Dero*qd^S=!Z$qXs -z-m-+}F!g`aD+h5JYU;noiVf!G|I^(T-Uzq;Z&9X*? -zTStaBI=plu9WK-<*-b!KOs-e8Fbc-!kJ9~;$PI$+HL)2po5Hd9`u72^U^ -zDy&$I(8VaXVik3we^_S4^g6C0Uun7k&AiP_d8Jmmm+JV-7(5y-3*DwVvCl+j7VLRo -zIATC6wqQlVrphZ;c~8~xKDF47@{)z_Ni)8it#m8uL|)Tgw9w@WI%SiU?jI(dvB5(3 -zh=_;&f|c$8!K*!Q#r|gUD$iMX?-4p1>%DX$KWEn|8+xZ(rkNgvz3J|()1T{2H&@u+ -zc-l*c^u5`%mHw0^+zeCRIxF1`qFbe`wPM$ccoHBvp7UWLfM+9i*aep&?AM(h&uc}+V=VH> -zL%0WP#d^tmtaKj>U9^Q(?0pf2vcQVHCom)1i{VTV{G@!lmgW?&A6$z=0K02x@UCYG -zBGXP?roMN1`Pjm({6Lldu!n|yTFpPG(jSUr%^*c|StFvOKj`&8P>q=A55%#Lsxcf< -zge4)H?H9P$UI>0qHSemzcKq8tydr;;)*AETimj0JH_ -zuja*78a_&@`O8(Buo~%{E3ARzx-#N0J(DBSO;7jQ2H5k}yby8FW_sda{L~`_8$b83 -z!fMJA(op?I;(e-`KaB_&H^i~E)%=MnY?vOZz_GHLcT^$5V*Dbq09#ON%=E<(cFGc3 -zDAT;*Vh!EATE1{A<{WN}<9Bbx!m(~TjQO@&<6020IkkK~Jhd5d{I6Scejv|KHmg>d -zN)jkuIo15GD($v7HWQRp##|5nNHxEs%9tYpx;)R8dOg`U@7AiKgs_ca8DakNysxmv -zBqQA)KP~Ze|Ah|oq!Bg_rA`*3qAZadbl=aC@dRWJ3tJeL7UrLRO;t+bS2L%`AsI8T -zUZ0lkzd8)Q&buK3dWre2F||gDr3|sz)i`IxG_S}9YAx0!##HN9`E107RC5_HHUM5h -z`8uYSN5iRH;f|NcS7F#xV3pgynqVv!9Y-u-qO2KXMbt$&5{1*hRvGUL2XuXFc@R9b -zB-2LF8A4b>{u;?$ay<4^67L2{Km4=D1R~(6cOSlZ#D@TkWU*D=IR-TTnGcfH5bz+F8UAXk2 -zULIggTWQd*MSC$ZMbNNSmo0Kg2&>tu#ab{{<Gsr7P=;PX|L|b`Y_ULml -z@w{m26XE3%;XJq17;NR~slt9+HIDsG?}Rw^%vSzvr9Lc9s#Kc%*=e9g0!g6q(ys=& -zSsJ}RVdW?DyV9km`Eb@}t#rBkUFp&WausE*3~|y9$=sXskIxRvNSDSFoJ^-0iF%Gb -zS%q^=;=DRR&lNzho;%ozdziG1^7@PT-oDl7XNfPu -zyLGFmh?D?9h;j*RmWdgD7VO5Y=6sXQf?dDW+`rQ!yW<1iYfMbJV8K$h@>?o#9>{MN -z?A9v5tDm=EvrJ4oXTh!%@l?(Vh{`I$7_*gMg0f(ow$R0FtuH=<3xH2qu)rD-H|_>3vzk7KQ-q$t! -zXa)B3f9(lR&A3!d0A8VnPf|0l#M0sbsXseZ!-rN%$Mi2{@Eop;ru=n8#@6L%8`f>a -zOa4LFLpN6P$68j+iS$XH%KB?r7pkh-#_k;&|A5`)OtI+>HWWnC6 -z6ccg!XC92b3j{+v`GZvLpiH+fzFEn$AWHjGkcjxbQAxu^kSL!Bl8#EWS=ixn03;Ou -z_DUK$lK;oTzsU?m`^fFzP|5!Sk@}}HzX5M?!(z@^+iF&PKrUEK7p$_%N;+NleP5Vc -zDk;UfwU1?<@;~srLOjNMULM4GLnSq6`ny)_Ie}^Kc(M8URA=NZuUM*vnX5|Wnd6pb -zhZT;mOvxRGyP3_^rbV>_5_542qT7jUoUzKtTEOTp*Wut7Wr%e}m?%isAw?z)(_E4p$e>%MJ5aHce -zqrB0ZZidLi+72%rc(1Jy{oi&kCguyTuF*OzynJ#6+QuQ(N{4)9aQ9nmP_?KWl$|oI -zD+{_X<-&m^>JT4(q44dIOaZC=V2RAhMzOt -zl?WHwvz}^x3W#1I0;ag0u1-H){V#E)9)5sU?hRJNRe*+W301`PD0)|dyi~;XYmc+U -z9oHiQ&|KcQ_Q@Q={wcb)Fc6cw8>Au8w@IVX1-(e$FAsw6$8H(QE95)U>$_JT2H*Fp -zY0LrrULxOV9$({K`3m^%sitfN-xtYujn}tZz81b6=)lv-xcWx;MhAO*mAB-R@c)OQ+ThJyI*q^=XzvoZ^*0Q -z@1ntrxFD*K{F%+`{~GKT%{Eo@cehB#!2b;V+1p#Z{+%#nG%Kp+t3+ZNPm%u$pZ`v> -zEYk82Hiz;R)hVv9mdeAGVHm@#_1YYMllfy_Kb*Vwq{WZlw*}3|vg#B^SfvZB`YM@B -z%$8J3Lrq|%EKStL_>1x(!dd8*g`wLDi!6kb>Geb3!D;d1H&HnAz2PW%W;i!{!clS& -zj-gUGc{X-yb%-7EkcTDiYbar{Q_LwYE>9ODusqC_9Ek~wQ0O?R9m{uiAA)WDC#so|K$yq5krSC~S -z)aQqw3cug|UBw%;+xs&`WH70}_JGVBA}eD2wp6bEB_+OqsnvWS!rvcZ)0fCBzM2n! -z>rkM+SVrYU7RT~|`(+#&F=g1Z`&f82w~^}o^}oS}#hJyK>T)ThCP@3DrzQw3^pT$G -z2Ou>sygKFbur*^3j}5B{;OEOt`HSS5p!PuJNRKJ)tD(o`tkmLcj|}}wA;b8pCvxls -z65?+a@|`I}xmyU)zVZoSKUPrzLi{wDeJ^@Zh+mc*`Tkjo;Llb3pvkM1%sl9prq7ev0k<^cuX5xsCrR4jD!$ttojYZAsEWTXq#1X}>>al> -zPl^srcU+^449qdM<)BCl+~eNTrYC-kdFUOCve3wb4{?|$T!$3l)E -zul#91_KC%;(XEPci_Bg%Rq3zXB4ZnMQx(?{uy(VI8gzRVcfq00mRY4IMzdsW(<*ip -zI~kJSCDpTdri_uv#22v!`dK`%pwVA=SsKyd_w*w@ucU -ze5_BDYc~p3_)R_D!`6A!ogt&YPgNfN$IT#h3##}sQg=G3`*an5nADvnv*jLjua|=k -z_prx20Kv-pXQ*9&QTNYKk6iDBjkkmp!Nzk2s3#WttQ3X0q*p~$mF+#kAdQ#;=GngZUSFQPAoT4z)Dt1H6uAxpD44d+>taUP`sv8@kGQ+nINb;&5Ii0NU?mps -z7N8}|Om+*b6&Cw9qPu6B)8 -z_a~}H#*l#*1d@Tz2C65nLMK})b``ruF6V=*QYy`MfS)QeE`>1ilwVCPt`h!XKK~!f -zj4{HWA1gzh7*b`77ETsi#lHcfxSf5f_?P4yC49}kdg|;7^amYTzq;3*d(TV!wi1U@ -z?vk;4(SK^JO-);-Iv=5;I6Q=k;+w%#6c+_kQIvwIDE{7`isIwpCs1v_*O$FtDXN*F -zO03NGijJZ*REv=#2lknXis8db-U&?(;RHIF?Wr^?793l_KCa}g@GxZAHOnSyzUTIN -ztInvJApI$`+It#RCj{laS?R4hskVFuH)!)p5)kbZ(5vc%36LB&I$n05noD_VQRPvQ1ZJ4cv -zNRwo?#v@Ils*|J@Rr0@^(t>5?^hm>ZVI^M*596X<(#*9vqsl2QV1q+s4N&gv?-vuQ4KKTD@j?#?t -z7P{Lic>;L#a~AC8O7U+~JL|#NO_e;cOuxvZ+vuP>{0onkIg1w~Zqe~M-@8UUEk(<> -zucr&+XUkXAr+jg3rM$o3yFJ`X}4U -zF}7W!Z9yY!Q%BiWUTWJlg1&e4^qi=D=J@Q;dk5uRPQ$5;g-_FS_yf`L5MG#EDW?@| -zno?|c3!gg3wldO&O-3uj1QKkUD#D!#XQ(ZxuPq1{&Ze4k(oyM1Zx=io+KrS~Pp6X)5HSLDU^H06OhoboY+wyl5%huIfD -z9ER_L?69S^%PnWyLm`pZkk$=bs5&(cThnk{x#)iBpL^0qt)C(tSQY+fHF>h?m>@Od -zJbSeWcfb5eb&irYm80Ep#nUc^80=yQzgKBsKJ_yFK&)>r2IqSW8*Q*If? -z44=ia$IFlk`oUN!Sn-o_IL=^AtxFQP40x(6$_mD90o -zA>`;}US-#X$jUQPurb`mu6GA&G{>^36sXc2%dUo4{q0!emsoZ?n6!7nq`!{HU>Ufs -z*w`dDlfFHcjVC5!Pi&AK<}l)IY@F#i#&|WBC6Ru+V<|gbPZ@@6Yjg@hLzFlh8{rnD -z)W>^O$BF!2vHb0o^ -z+*_yKdj_R9JKsJgaF^YdvkJSS66J)BDPbws(CrdtH9lf0Bq5K^|6M1Z%l|8$%S|5S -zoS$^cy1MQQ`+FfL^1qT3=??p!f=B(YcvLsfKEZSIf5mgN$&>y%@xTl@M-%5IELAS< -z!2V{jvuoi@czJ*#U6}J*~0q*!MSb5-(uyxfh=_ksvk^$-{+Oj -zuH3?VN(7C5ub>$Z<3fc+Ac4(rj>@U=A^XN{*}K1Z|Cr@8>DiwR+CsTbUr16+yT+OJ -zTHuw7H0ds*i7@4ED;94G*Sr|bhOyW!$~~5LP6|kk+L9i<1qCB&tRrIdh`{WzIPy-E -zFZ|ohF=nDB8Y`zp5Jhs9_uo7V+rqyr!Cz4=mVbdaPEZ@lq%F!{V-E%7g%I(*-xm$< -zz>x?jWOW-ui#N)^iWbnW8s@)+e*hlm@DAm6cf8^*P}$6DDM@y8IiKVl#c`uEWqaxu<;OF^4wr_3VtjD!*wDl)L^bdqQr6cFN=a>3@ofaYIM$DTMiaGn-DKd4#r-bhEvyJXc*c4p$Ldv_e4m^t# -zC~vk2aj%gO#O9XF06IoS9v*>sb^0Z>mpjtcw77xrO}p!TJd@+!cc%qwF6WDpceER0 -z(Qq9%4>v%5Fb3Fo1BRbc-{Rrxkr8;gF*TOemGi9twdvlpPb#A@b9Ui)g|zq%{Dy>v -z_$MzyBK4Bv$+n`NW99s23Mnm?y;9C^!Y4#}bVm@I%J~ZK5T4MDhK+1_xpI{y -z|5CPR|0p*md$IYpmjXT-i+#vYd0>hZY0GA1krQz`Lg2=*yd~vJmX;q*4x8Y&?18`c -zSu=J#=61Bjv59L!rOMnFLXU66p|l-qrs$v8qGsRn{E-b|EB$Pe{je*n!jEN@ulBR` -zOQ~GFx^IfiZG%&64km1-WA-ON>gIR&)b3ZJ{mC0A{Y~K9^~E_@zckoF -z*QZ>xAA`Jf%wEoQNYkTGRQk_y-dIHZMQ#EWsGMyG#jpXI3+wyUTbhJhYHl_C^_R!l&Vr -z1^FMA@rPVkZ`9AC^FHd5V!d)-3)WpG{>>Xf7Hk)~-8f%A(5oBdb&9q^v%9hRw)vNp -zWU6f5dYG=)xXt>bTV?6Z;D;3BAf8%@%bD7`a3cicY -zZ-;g+=0GxgNx1a0UOvdeR1NZlcE*iSd!XNO4voSOm}M>F5iTVpHu^j5CCZT}kp+(! -zoAIgLc1{|p-G5G+sN8?9#;@J)9lt|<7=P$1&Uti#?0hqQPk6&)xn-iZI^eJU976-G -za!g(x2-lCtuz$E64&x_M^-<(aWM6(? -zS&e^PLqYwEk1NldBAYXuHCx8B%ZgnEu1$~ERpMz=a^oxF5??EB4%=Vozdt?baCMmR -zczknuMe16E9G}0DmXqD_C=3S7CU4=kTs{Ozt*C -z>5rK?Qnc}W55qd68_SHNF*gKb5aaa6Om99O%JNt#;EIfX*pVAzoIZu~YqBGP$f2A* -z6*V^d>Mu!8%ys^`j5k7r@l{Orkf}KO45y%97Ly%5sU|?ljKw2822AJCO~C6m{qa-u -z($d9CKZ{p=@j_$ysqFCJoO8t`iN(d^j5)EGgVaNjERau0&FV{xf5!O6=CEr`33(_7 -zytWo?!_YGAGYcI}SFlUUl!F$GT`Y3HH29Fed|0|LcK$#3QU^}Dp -z$00@Q_JqTJ)g=l0JL>k|rM&Hy1-kEbo=}1g-VdoS@V9xakc4bKNcM8!=?i -zyJJ?`Y)V7S<~3UX(_>I6pUXL88P-Dr!W41s@EQV6DjK8X__ -zlJ;R7Wd{Qp`=L1eMu{@6R`HsI2tE)-432k@C!~=5#g9oBDP#TI>=h>aX00J^s-5=P -zhoF>faf1dIGP^=AwvpM$ZrS1Z-^utACLP27tELz>x`h7?4!Qt<(qecCAH`5XHM;Gx -z82eHjF3NE>9dsy&c@C1u*Sy?oO}Z!s{5{M1w2WqFnLJ6Hg~S7mDwN&b*O~ -zoC|Vwarq#0*>^pYg7ae7&l82u{l(;ezxhq+@4BDD{^E7kHRe~GG=}f4`+g;Sfhmb@ -zxN$AmDW08pq7z$RdnY={^t~WY;OoV6qj=J8eAj{(^<8!}w4!;93_^wW9yPQbz?8t# -zEKBDnn6w%DEz3l95&3b5ab=9^2vw8&;udr^PuFJR_*o1qqNgcWf=XI5^>^{VLTCK5 -zV!AU+DM4k-L^Fa3Ay*8WrBeqBe>7Q))VJ^g#nid5PCPjdIZ(vP*Csl9pEkn7Hmt+N -z--IkI0DL)Svx{A$^EZmMx)|j=Hiqfv9N5l{BY7sb7%M_d`m1!_Qj9`W93u)79fq$P -zuf$mIl49M(d{42|2K_e1us4gvP!sxXh+%IOyOq^nh+$pDqEUpl&wFiIDQ5#@E8h -z2I$7K?n(xjxz{j6*Vp?nWYHkqtY6l7c*PnRKV2!b)K8%Mt<)U<3o;4=vye$=N)J22 -zCOU?r^opLX2>X5*nq5_*Jmm|A&niZJKB2@|Yr#s3(eD^nqOb8`!iMb9675MJ#v!kb -zjVw`CTd-S-u_i|;F;@C8VPz(l=ucQMmkn#Rqe`?DK8%eo5iwTsV@^%%7q_R>b5^p$ -zv?og};omd#V)9~6&FvT0`7iz`)L$^HDq9{Stu7zmFRrB*g|L6#_R_-brK`eYUI-t- -zww3a2rRCvrV|ZletEKy1E!`d7|KsprGMMgfqXtB`ir))hV -zo9pvpq)D!fAc%O1>3@xZ-CP-gSVCRzGVY9VWejk)U2R3FDW${4srtAdTyK4c%1E5y -zRXDkHykCLC=`1;#>*z9)e@JZU7&Zyrsg&fqY6A4%dJ-KSH;1E^68VM|zd~0RZl$b9 -z=}Y&(8^4_Go{NSkc&eX`8;p%8EsN?iskN>?Re#j%_s+NJcSo>dD*9joJ`8F0& -z+7!Hbtz(G&9Y1#D742M4_uSQfQt+_BVeFz(n)(S|H%$pR7t{ldaYiED&)M$R)tSQj -zmQwXQ#5TRsc^fW7*!|*9%wEbDz5@Hrh&eWw`Iqu+a!il;Yz^xv;djAdOp8G;;r9~$ -zS2&bu=pfYi%?N=soV3zCu_!Kd-J|SuiT2xB5uC{H`NPJBB>hs-tJa}%An+gm1+Vcx -zbyG0^wxq71xG99x57F{*L$$E&sf>O}UzQ{~iaoU#>;2(^t1_Xt -z^&`@a>hFF@6}{q`XRGn!dP(#QJu~VCzl_d^cQ6mpWH%aPr%$=%*`M8ZOM1B^?KQdG -ze@g&vcAP%;+V#J{gXE13&f8FuJGOYd&=ahfqK!I3FC`K$MZv>}p|6p6vx!N2h(9+Fv6yUyT8 -zJC4k`DQYSXuB0@f`JASgl{j~`Lm1k>hL_1h8{(~xqvMB-?UyvKWTM02j}D=?{fl%3 -zs^QtHW2u@($Iz*nvbuDr>Kra@ScL=aie&ox7!^IFJbYgCsK)SE;uB5ojU~ntl)=JN{B&klmnegTlNY{>dMTwui4;zDCE7NuTh=Z%ofAs=RFQXD)IF=0=Nyk?RYQ>*>C>5kYVtFUf -z@B@cyiv1n9mA*FDl)+K6qo&;Q{9)4sWG^9cw%i_Ey-Xc4RULUFB0prfx=BIghsf&O -z;p#n#y3En_){m_D6`ajC5;M3EZdZ&jY7cAt-?1Bqe4ih=KS8ixeBv4`fBk*&Dax`4 -z4-1=uqld2=GD#Y;`E507h&pPJdRv6LC`|pwML4f6s-OB|kkqz23ia9Md%9nw@bn|T -zK)w;5anHdtUYW*qGD*BcimD!|Lh+G9)oDZ2xr5YuBGhGJ>Kc+eq@TKp(C$cdtl4+tGsC6^*!Y8pVBM<}LH(!DM=lON)W_si#)a=m-|Z24AYB<7zV6|7 -zg2+=_IPgH~#~1gd4EGKOCUSNHsp*H+^>3P!yP_Qx`BA}oapzkuxiuO6b_G)TQGLVX}i%_rjw?5D0L;|wI@eAfqO$fC^6R^ADgCZUA&iK9FMWR_Cg -zQhdBMtj1426|>|TSK(AwiZd)d%|+3o3FVnL6=Ym&+cX@lPn&vjtoM&z>Ja5b~&40!U(VC0v` -zi9za+Yt@lc)QPF;w5!y)lhk|0tIIA^*CbQM9HVX;Nf}d7D=)#BxtIVswmYhi^!vwA -z{@JY(%4uzA5UOE+ttFI1cSd@%m9{h#sUBi12^BSmJ8|@7G7erg>ngIN9ts`WT2uru -z(xFa_Ri{O(b1zZviBy*jR53#i6++ZaeMtq86FV=U=po;}V@JX&p9|UTc2Cml$zi+B -zp`M>Fa+(p0JWOeHv>(zaL5+~rg#VO#C-$kaYm<~ag3%Da(o?+6S>spNO6j5;idk}) -z6@I}eRSq$4s?74G$~ciK8QxS0D43YveG?M&)0oXSQK}^Vuc_j#m;N8d-aRg=;{PAt -zU0{LTL0FI#c9+YdXH`TKwDS5+)G$#iOfh|DZ!yoBrg^D%7sKLWh+GsF01MHufG%FrLhG2C6{4^T$bO%57OcMC&*$;`{jsxi=K7k~%)I6`uh(m4SbNZP -zk=E&JVT|FmKqS}vbRQf=(AKki$2!-2UWfMp%=Gy!ecWD=*PytBRg)v%qvU+P=23E^ -z)F%RVFFs=pABJKWJK-UsvEFpy1&NyJLl2E!jzMYdztpvlO*V(o(iWgJF+SOg1QBQR -zQ^_fKd=TRWq##FpvOpaCT|vih7&HNf(7(tfrIo}`N_>4~Kw_mOo-n4(1xa#G3|*t@$IURxw9|Kz6wg4b&S@=pY?s -zeh*4a;<6LCw6vlkle?tQImu131B}DK*oMJPVmz0<<$sO89+dcBj<^>(r=<<#6zBGO -ztUW-_uR+TlvwH|;*Du6&=sSb&7y~oP@GxUo9AaRD4FA=k#lFJ~yN9C1zMkYFNXi^v -z*VlcW4Ik5)9;y`C`-kr}7tQf?dEHlLMDTYPA7@pnt%||%YAy|CJXMV@7_I{6Wu3!q -zRaAXUNmiw7r2%lJ#Es%G8yqM -zO)B9l?n7D$NXnzbd$yt#oBwsjdk@u55wVSZL|>XZDEeBS!Q~!o_>RY`>R?3ntsk -z>dY-rR2_EUV#cvxcBhxo#l6y1@WQ&HzA%IWJ(**(yrwIRBgIulSO=hC@xE4U(3)() -zXK+bz#b!i_C8dVL9okq>e1NMO?u--jcrYynd|#ic+*;~GX9m6?k7=a~xGR~qG!d)t -za>ZcOv`bEGix|}uc`or4RmYhT|U#ZnB;nEbg3@Iip%K -z@P)OCX;x*tMvFfqXJtOdk+kEm!muw$npcDN{Nd&35@+TeuhdPW;u1oF|IR~1G~Di< -zr3gQ19^A0#A_n#$;CJ5fDtDn%hdpL*aw+#csiCWy6aNTprn4aPDQ5%`*=!yUZJ2 -z;yliGZ($&t`gw6Ge7_-bFT -z=2ew0?02*?=pu{+e;j~ghBdq%dYP! -z8L5$(^($!e2k92iIcRi|v%Zt&_#X+iu*G}MGAFuh8ixBCCeW_Jq-FiR_0i@~N>7o| -z+d(DS^CRdD29Afx2kK^vK{+Y!=Bz)DIt-yh4O3LKCW0jai+A5ix_FD_LryLe{6(5e -zVT_Y#EY^wU`U&RNa5H)TN~BX;q^u!QS{*5|C?w2;yPf;(HqE~&JVGuii5X@lLk*g7 -zhGk<6YeyNjFm$~RHWUTW_1e!6>RS@33oX@#E=vZDrP}KeoHB%xem2TG{_e6I5blJu -z@e+SW#9z7s#~wr28ASPWcfD$`)p{Q7d4NkHLyY$}getP#&3bpUnZCfIcQZSBLQF?N -zg;~>bg+UW;ST@eEc8p;Q-K=Sb8;a;=O`|o0`jsrBybH8Qdi#vi!6H!|FFs4ojV-1M -z9p+yF>Yn>F92$mk-e991l+ADZ|5-Mp8!Vfh+2 -z7Y9bx?-TDfPZ}{fA;N3q6V8=VW^v$1ADKF>KX`u~agXm5xdBHX8fL3#X71!ZI^_9# -z-eX~A&3#V4bi!SXCZ-P#e-Fz_Y}$R|vSgu8rAw`pw|l>+`aV{|y|Rz_u78cwyb0RH -z#>X#5{Z8fe>t&lh+DCHd_-G9|e&pjNK0bzLTnsPBjIa2t7;fH3r6;(-deHm2NRtLv -zx#m}Dv7Kb=aF^;O7wgV_XPl|x5$xL+2tnEZOvitUp^bFg9;9PWAOz_Q!$ba=ROoXg -zee*$5rXL}6uE8J}9M1K8A6L>->LW9L_!1ond>~ytJ`&e>(6&BL8VJvI&t`~L;MMUe -zFUV8;z&H|cx1gFQJP-9y4=(-nV7lxS0G;Fzck%ODB7E!_i{ZA14=UF1?4Ey`)G -zdlG-*HgC3hNAeEz&5~U4sxwDAznG`@tvun?_#?g#z*FgXEsJ>bxcrgT%6VMmI`Te -zB@MeJ!c)POu2U9Ina!5^Ul5i<1!F@W+Y`vlWI_u7+*S -zhFmI`o2`nTmGN%s2(aQwev%zhJhqB=m!&3u$%Wm3PD-v9>FEzk)Q)qm4gCP!hLmq|={L*Owj -zgHns#+mG(08`*CAfsk=zXp@-cIZ3scrusRm_$uj#G(|ek%HOeHlb&B`I=A;}D75LlT7p2=0+CZSz~;)w-66MjkfZN_gUehcxN -ziC+YMA^3UYC!?T({&>*jcN=Rs=r!@tL>$dnPwZT=*YGiv#0`e -zChQDV62vFPew8gIOj_fE(&#*W33m@4*vnjdP!{xg`}dMDf=&02`u6P=1ff(0n6^+HAZ$Gc&whw*?=Wv@5x6Sd%oI`Tt7ZY+@_p%oUfJYb? -z57_C|NBnm*#~Y6FzkN_X!d~5L{qJ7fC;N>K`o2vl@*VJ}Z8{vP+ql>IGGZ8fxNz$o -zF-+^<^S-!O6cGwvi-D*Qb|`o4+zSrCpr8B-M(ON*yCK34miUI5k%o}DxHrKWkBc3e -z2|i2KD3y@u)0|0iSQ58*FE#UIzC_gyq9k;IRl_9M*%Nk_t`ofS&r)LDDX;N2L2$?Cfodc=o|h0p%#$NK2PY_Gil -zHPLIYX>I~`=5>$2dsjZSF4 -zL_gcsBp9M)Q1-%a8J*Z(K?9-YXFBiH%2;=>_K>&Xz)X_a9`_T)jLDEjGzl~7W<=*K -zci!&(jq4i)|*hF5XJ3$0#-FxmJ}SAVp~+%j>- -zB~{%CFT5~@_||cSL|GjpYgk`ZK+|(W3r%aYvG(`g4Oh~FB8Dfj{ddCLHh$;@3s|%G -ziesfL+ZoxC0Hseq(kwKlaV>l3Y`f&lmg%^!2K;;}9kb1QsN`HYpo=UnM&jy*eq{p| -zDn-NyYts96*tXV{Clj}-9mxq5Dxx$`2nMz+m3H$4uY?J~mWKNF>#WtoZ#Te$!!gNSUi=mpWp8ld_1}C7#=ta-dN#9lr6`TtHiH1ymCmt -z8t@d+F;!iK7g7Kg%PK4T9VQCVXg`!_XFxP4f1mO)qJ7VvJ%agfG)G^&r$AOhA4yAf -zx4kICA(aBGII-vZ2WSP?QINDhICxYtU*R$x8%4Ilr4}+G<^z{O8W<^I<^7m?i3bYO -zD)>V#!4*m73Ufz=ikU_WLA?u{jNYz7>)qvwC=6g1_Gw$jvU!Wi#PPH1V2hOa)}HZ? -zo6}~`gt{2~8q$MSyuQa_?vq7n#jAU)Q;MLP&Z3ha@z!;P(7P`yWOM%-x7@lt*3S!Z -ziY`Y--wIx{XF&fQVs^|A&bY@qrU*19TX#rffYd%T)G#H?5DgZ7ykTvGVar5A+9Q-d -zcIQSH@)P?h)eqT^aT;RYN*(alA?7CPz*Z`R3!}~q>SR&p&H!D6NfS4JkC65^xSrTU -z^&tEX9q9}X9q*W{Q?ZVWE@V^x)(_T4X`4nAGTZ;kYh73Xn}f-Fgm?c618yGPWA!h@ -z%joY6(n6~*KDqIG*yMkt8?(pydm-%g;~%8Ej!(|8M@aZbI{h9|Da~vdOy|&1>-Mm3 -z|CNs0S|X~97$}u3%0*%fRd8oYV6ye5u2G7c0^w!f)U~SUBOnc98{+@$=%Y4Co$DJ& -zV7o}f8rC;B>N0w}b9FT9W?+Q-6u!T32)rIGp(BiWUPPCu?{+DH$%(K5%L5)v`>14x -zl)XG03q<7*Y9lsRGyv1FPf#mCtTsZ7=n-Baj&C-?5iyMq8Sn_329mPtMb8;2y^e%A -z(@%}jDjhK5nisAJkkQqPU##N_2K>Sz9W&LLt15tjfnd~e?-x+(11#9j=tvI?9Pyl$ -zf_1c1Jvptg+eDc^8h@fGUIwr)>1i -zI3-_Xhk?{KL@`JEkSOa8PQGcJ`Oy3L_^ppY*$U=iIBcj7w-G?H%SAxg{@bYGEb0=z -zBP3M#n%25F0O^2R1>X+`&#fYMhXGXH&dwIRltw4`Sc -ziMt_44=(^9HnNZJ>t@I4xV^>Jjrsf%9amC}K4Rc0D#m={V|849F>u7d^Jo#K8Ve*o -zTr^(^1acFK!jvrj!;5gJ@fjWW5Oy#?AUC#%%o7&sNGQL}8`bRq-k2=34R}2Uj(`aV -z8ierxm}NXLWVna(h3#0P_&RFM=0hX>@1pxCO=bSA$8;Csz;u-G?f3azPC`CR0L-B| -z{LuTrXcz5(njm3BFS3s8Wpp-XoQ!`&#@#+DhiJke@~8A%!pY(X=Muc~ -z+3^lq5Qpdetj;Aw$9yuJ^i*@b@qJ*Ht`DK(g%IXvco0Z%1BKUsk9`h442zlKpYS*o -zCv;XYuHKHGa`f98)j+>7FRB2avp1@_S&R}Fd%}3&MzM|v&-*W|N9O6$O4$?GIzwre -zE^cre%!!Q26Z_H}ffmdvAz~m?8F%`pWr$eKDgA}w@E?kCw845IPk2-p(&qK3sRL^P -z+N0pF*hoN<`Q%8L_IYq^#nx~0V1Ta2WQde~N*{&|c)D -zhh(VPG)cze%uY!P3{pQ1YJLQR?&Bb?p%?>lRWDah#~1UEj{B63FCJs?crlsBj6-3< -z^B%mqFQbAxf;tj}Fr8c`;Xa{$eyooBuz0?bAEV=}#bHWjjE?RjWd -zGw)|_0)t(ky2G3Pv8O%hMOQc3@9f;FVkQ*Iq)jgr>mKQJl`MB%9W=Dkx!OenNy@R| -zI9Ji&BE`O}o`94(58aWw3)QtdJo^<*?RbC}Ll?>VMhvW?v{yqTndd9WV30(IPa>k`r2L0Yv -zr%E~NGemHb2s7?7!-oi=Qf@jrMIbr^7&R^ya}j^pEucf06MN)$2ReieK~cvj*>Dbs -zDjz%(9S%tzGI8#O82I?#USwtR1f`C9x5&COkA)|>$hsm=Q0TbCBI}p=3_R}?S><`W -zyN=shWYy#`@C2h2-i=OhN{3e37eLS9k#zc=`V9|urQOMUI&bsvWHwZ)JX$q8nSV%X)X136bjC%b(uMv% -z{8+pNt2pYfbP*bLwHOcpYyn>(H4ODJj0iHU(itZD8)gqTF7jcPLiU6uz#EGk6H4i{ -z9#Vz2FM4$uEn&&Hh^4`7Z^P0x#QIYa@vJ&*wiHXMi>1QwV9eu79s~5bOjhGf+fD+7 -zRX706``$&79(0k>d1DlgK)H;P)^O8<`>IGB%vZEV4e-NEbkKNF4`JC)Z*&sNVON0q -zvJrRLa2+3OH%`GqhR0m#(!5vN2y>)>PeMVE5I7Y>eh4-@?aeJ-CiM -zVJikBtMS0)Po=vfPC|&tDE$o~GMuxwRu)w{HT)n;Du(G?`vU`a`Z!_(7P{G3O({jF -z@-Rk>s(-J@YC&?7n6^+==rD#ye=8ceMQo&)c9Br&0WgLEF-$W>48{wHz)l-BcB`OX -zppz3e>W@k}6BbG`-*loMysAgmxn#Cki}5DXQ8JeQQi~Dtr;Xb}NfUcX%e_HKlW5`96- -z*Y|=VvUC9#&T1(+T}l1?IW4yWi#h+jmQLWc_qmcn>&v^Dvs!Li0geA&%S|bmuVnB) -zsUS?rf2ZXhDZt@F{7)>v^!QfGMHGN)f2)PGehA5U0TgH+kxS8jA)4|g|FJW(0S|xO -zLzt;WMM0P$i@>x}TutG}-$(FG8W$6^K*7@&f7$Q*Dj^3E&i6HZ-j9K! -zs5_X9-5a0cWQ=g(W)w9#ON=vJxW|exIj%JHbN?zL9_^Jm&0DzALT2TCy6C9ypyqpE -zdqDsGxXnERG*NU-1UC7TT6Eb=%E~F37jL8Ily}7ic{6Oputd(|BOz}_&rO4z_&BCD -zc}W-wLpn%@Sbve5I**}NsDlLqhSqO=5K>RThnL=CH|eRB3IB!79-8}Ru&zQl1H&7c -zuvj!cbBqn4gK)p(Qv}zcVFVAP1&V)(fpl6?UU{-!$khTJeIVn+NB8h1u8>64=;!<( -ziyAGdbY?#RO3J0=_$e5uu(rU~j>0w%iT=;RB>@XS%0YDrNI8tA|MsyVut6r(hAb($ -z)GCOne@5ycq(#atKOlRhvm{`K4;-++X_4!_-G` -z7(3A6VOGJ@`a)r?>xHv`4vQ0@YX)?uRP4X|Yh5d$Rp(dq&ohq9tD%~a{SMsWKAX=# -z_rV1SwA++S+Th+xe^d--cEETM3**lsvN8VjAB*Aq_I}8Zr_e?GW6%-)FeN6(u|)Gn -zD1(2*#>ZiRVkouKa<%N`K(d8Rf#_T8Q>U0+g}RAJE|Re>bNq(lOw&Lg@cSKTpM&KT -z+mcJfsZC51@V7L~A_vm`N`ozrR#+?^vbxRf1bD*M{{Jbb5dF`b(2*Vr{J#^@Rh=#V -z{{}-i_Rlns_KEW|azmkdvpcuD!1`h?zgx@Y6jXzm%GGk&1(Z<1^CXxcAqPX5xkw5A -z3CaUWgSatth_fawH##31hHNc2G9T0k|4cqM3|U$(5=<3~e||=c*_uKpeqoUM86@;;V^FNyflGvCttkW;cVt -z4TSTXwAj@yDzF-Iaf9(j+$e$a@sf7pn8E~V*Gz>-vgVoG_5wz8e{d{f3g5TaDhxhg -zUbeEDAmUdk3ogrHjw^`wnveNj8W-|SptE|#5jTHw32r2v;Bob#ag -z9?R9`(k+&-L`#MlyrmaSRIel4>?%;*l~O(Sc5a7W5;Q~dD|bEIFm#wP#6>u$O>vTx -zNo!QiGnqmy)(Lj6mh0SYEy-bw+J=UHlowieFo*ZHHLp^(scV<3la`a7447^)=e1an -zNyf^!04{U4u$bOQ>5B7?ByjxO$wu!X8LLSVbD)bZe_MFIi+hhs7f1#Py5zd?r<}I9 -zAa;S4+XqbGx<`EwKVQr3Eud#&$pT@XR%rrLpvWpq7R+5-VL^QQ#)HY0IB+$r&}Gr% -zic9A+@=LPcomdY2M93jhH2zQy!*`)E%`4Fb?}_zJ5%zZp$7!#P*76f{mN>oosGG(7 -zDSs8YV!3Sv)>S#Uo-#m~t>uymtk31(rOp~h8lfy=WD=Dx(MZXs0uru)RCiGg|h63x8v{(=JF>oUV)_LLw|1WL)AMuw~{FuY7 -zE3kIv;}JC~9I5 -zi#SBQ-evSx!f7|^x-y(!c9gW*Vd4nmh-{qH<6-h)ev@lSJ)KYOU!vqSTs4tpVNdb -zt1fL}@V~a&Y~jyp!nRafA{hLq;aC*^y(SE7*r&|*nnqvA#g)2HVO3XskI53_RgY4+ULe+7vx-Xr*n(@*sMHI_+O`EbgW@rtL -z!6i6?P=!vz{--O9SYp&QR-MKXEUfIJxbg6rrUCJ2(_GQ(Gx{y&^&sqQtHxq}8APH* -zL(=#&n)nP$j9*PXNdub5`~jH0)`0qcsj2tMwZv#@TFF4MNIq?u^>#jk|Fzc6e7;#T -zZHx6*K7;=>Yezo+rG`vt{WE`ajIQQ#{U7-v@+suZX^m2deR=q_zQ&y(((Jx2a>M#- -zKHr31wO-9qnexHM_|bLy -zg|6mbFe1vrNgvK%w;W1l$9Y^lq3hufYA}{8F^|9!A##1SUaRXFz;A`d=p8`E&HG$p -zK2=5Klb4|InTzJ}(CC>!EJ*!=EleaIf=&Cu>2WGS;G4PVwjckkq6S>sHJ2`B8Lv%air -zhzkT7%k+(@{n+ru1@bYNJXb!b+Z+=-?S6wkaC1yx&8?Q^7uxh*?Vu5Li>i(Xw_oyV -zxfpzTW3IVbg+P2F5N*O#Y>4v`_GqR>SF3{AJ(xkOrj5H2Ir~cF6IUYVUx|GDO61%t -zkqfRwK7A!Jrdl+*rntby2K)RWCqH#H;T7>IjYsWao+m$(?-@cu8xPx?pM!C;BIRTI -z9#5ia{Lo=vMbg-jz!e?kf>$ti*@V@Nxvk-TGSMX%EW=NjB&->%0Q+1w2D-wA@p(`d -z*_W5(EYVGP)bF9O&K3-LrKDmwUms9ScO#D$-)wM}JgOV|R}^b~5Jf8!K83U!e;)*J -zUAwI(v)SJ@-0j_z1HGx?I(AbI6rMkJ0~r1{4YxBFJSPKBM(%tie?!Bi<%TI4{HKC} -z;IC`Alw1hz@V_k=o4jinqwH{q+4w(usjQ#5-HR#Gi<3_X3h5Lpe#*|Q!CV&LXoQQM -zxIE$wD#5=5ANzVA`^LbY4Ymd+@~n^Ku!4qb$g?)&#SbC(+ly3??lif#XHX@|4tIwd -z_!QM^?r^tU@H$*{@m38L2$>$**1m(LN!qZj#c@y4hHZ{dxD5|&ZwIM|I(Nf%M-dGh -zwmUxIS{{10y@7kVNYc%%ChLo1nU>dR^#qr51Y+O;E_4#b( -zf`%)|vu@1h&uh5dc@%^28UIIGdeX4AbblNv<#wtRXJmzihnGqK-; -zNb^7FnQPchvxO?Wb?w*_3FG>W(Zj9bc@UEgH3F1=SI`b&0{|LE1fkeZ;hC|PQ@qPv)lf8#ioIs|NY-37?h*xP+P-c$=5A~V -za=PX25-h`s(!5Anp81L7vZ2hL?y?cg-tIDIHoLoQwgBHmeqT51qXAcRNGyk$Ld%I} -zgFE?I-KGoOScF4qWg37C8PdNnD+LRkpYZjt`#Z%=^zZgydfrWxL1>mr1v3%|Q|Sce -z=mehH$4?GyPJ;NxHPEpIF$m?|ChiW-8+_htG54tNdR2ekqb~MRHmK)&Wqkwjyr=o+ -zI`vLZ;tIxcYT)a>(4}H4$&SZniZAQ#G;g)#cp6s`^uZwj#~kPei=2eJH{E&+2niR8xUl`rKllG4H2=1_nxtQ-m6aYr4WqqZP*`NccM;x -z$h(cZ(*i(hy&;uv2P|KClhH)mY3}K`U8nXOnq+QDNUl?l8fxNZWaZSUcX(OMX7m0! -zb&qGIcrJGe8{Z8Xd{@98g<;XdhE);Ofa{SYw@Cd8&JwlFQv~E~i&h(*dS2E^;-FhO`P>!s^Dp?tgnE?%dr@ -zZ@G^cE+%jiGSMf(&Q5Sj?zqgONm{FDXzs5&k_@Xj7)BZ1i!w%F7cX~`7#2NZST#ZU -zeM?bttvhJ%cJvh=jBaI8X#WMF|Cnu^AR1fPyzZz&b}V2&VO2}P?Ndr>81)j)pAtpm -z1vv80?Ibkn9JHM&^>(s03hJpqOJkq;{hX9qdnFD*1}J;F9~*65wY#cSMbx1!DWC5AAI$X7FBQYUodVdN)yK)g)u1izQW7^wHA?XH|uM7|=myhCsUO=kB)7-EBUe++55( -zu^VbH;!f?o6lau>IbSUzZ6(q%&5u?(YbzRi?G=S7f7q>$?Uvu6G|<|;3kTz`o7nXl -zSbWMb`nv5RaYEd58F$lU{R0Wumo(O!_+Oylel_uH9Z5rRE7BR~u@HG<#ND=w7Bp3M -z=wPZL@}aa1;Q-X9~F17cg=m1yTk=0otDnDM<4w2+F1_5X-apZ{81qC_*Qwqx+A)GAX=yHnO67 -zgjLIfP&3f62Z|aNzZe!&n@Om5_&!g7fPO!HH@2ewzV?DE;O>UG(CWRAUDoWy= -z0}J8~*rdd_dGh3NwTn%ZFG&N-;xdkwMb^Zek+DCCNT(Lwz?;y}TMHV<9t$9D64?xM -zmNhHOIV(A5x*JX=a@%$_nBT6bsj3P;9Ht^J+QqQ6t@x0B9_j>S#O1(mM7el;xW3{< -zU0wLqo1Ns~l7ou2FS?g2B_1=zkGB%JZpc;EnXut?xljL5S1~~nh~RL^huq;%#Z|Q{ -z4#sJXQGNCmi|c?h9^||P2Aah<&^gW(ty0d`F3$Ltq%JADld%fbC}+pMYNxh_%%N^- -zP7kVG3xj1w3Z4d?z((D*Lw3u>0_=VVoMA*e`)@!W=q1Y858&^VB9+dK3yJg1 -zzOZYo6WA1s|7lB%1Rk8V8LM3LpKD6}k_9ea(RbM0DOr{1feHMKsg -z&^&O>puMN3Gfvg~Jobp5jGv@Z&L{fUKb6}A3p;aj9ro1+lglr&rT2+P_T1dm%bj|x -z$W^Ism!HT@u-7F<<=(rMX7Bi8R>!4T%ASrNlPf*W$M6>dEOA4PkINdzNsNnSQDY_Q -z@7xW(^9_uP<%|2V7r^a8-eX;xQ$H#9u*ThkdEK$3rf}vb`x7xyf!L$-k7-+%=3E|`dsY+lt44V%G&k|t -z9GgbAy(o|ym76^(w@gvsUZ^V9qdbUA>zUjdWfM82KcZ^VKPXSdT(HZ3!`=J{VVd#J -ze;>fK1uRgc;0zuO6!5r)ocS(5XhC3`e&~7EvRl@nx#sukj#jV_Ig!AUZiCv_U{Dwz -z(S;tZj&zc{NJ39HM9Q#;gkETil%gL)dpaW}^Ilg@&>ArUV)y@K5sja-BX?Pro=#Fp -zDNDTcO9$Ocyq{Ak8(%OhzYJ7{PD$kf9AQ`1-^i&uR#l^ASNl77iSR2qP%Pg8y-&QA -z6Kta0StuQeAxSn}%pvkpj6Y@FzhRhB<8&y!QYm~6@QL>6V{IkMGAUno4YE8}hFoI} -z*JvdQw1@dD02Yig{ur)#pmOgoM0yh5ltZQA%ohRWPTG)p6O1!)GX&@i-KDTul=1#g -zw@`Eog2*28fnauvKRBO5w@A;lUS88nWHHS-#Cfg@!4zi$a9~O@bB6g?VpY||stMzb -z)z^s(4`VgC%$qIwhQq$%l*@t7zX9PUFKwyFV@4j!8Ra}aLNz{O{P-B*=?UYj -zo>9S_>^yMihW?|RR9SNYY%R@M?-k0_a|hOt@b`0`aGEhHa>laAj@hcGL&oF5ff*se -zIBhywJoZ3&vgX_3wre#^1_VFtbSean?=sHxi*8vZ -z9)tHxLx{H_!qc$udBe&$!_1cn4kUy1fL6>czSEkWlf5hFhzk4M9#k%q@78QvXDp=kWsp8YENO-$L2 -zXY=AaPr5vm|>0QmIE1lWR -zy@pT%4ZnVM&S)gWo$q=!N4y6HYro4*+#U3SiT~Z-94zYC#C^;2run!mNVxi-@8~DD -z)WrPe|Ce1XRc&yKOv3entaQDnLVxZl6MyY*Vb}*nn`ne^`9XOIi!xEWKEC~LVTmvA -z#3bTonnX=S$g|%IKhju~8~!4V33aK#$}9rvd_w?(Lu8`an{Fb6P_fopC}CR5VQtdc7d|7*igX13V6LlxHVOyGFSM7@iKDeh*Up$XQA30@WxeD${kd@lt9*wZkNP>VR! -z*S*p`ywNnC>b}4(Q2r-TBJ|tn;T+-103_UE6SsfQcd0|My%n}o)F;D1ep*2nvpb>mav(Y@Oc1V -zXFH&|*YSc%z1f*^!)KhBQeHp(Gs*mEcgZAV?-*{U1v -zsed493u!=}`MB>a6By@t?v+6dL -z37v3Hhvz;jM$vcp(jRL;Cty7gO>8T86~%=+%(WOoM}h*5fH9$9GakSx&t}Bqn{^e5}Ju_xU~hy*u(bkmr1ys3|`;My_YRcKMyc8$VMw?Jz#l;`p~GOtXRZ7+Ht*zVtT00 -z6rN}v{465te}Hs86De@k7F1f -zrD)kAe1>QN!y5Jx)InG%OZ!atmgb`)PCyA7Rvh58|5@bFP&9+O(V9NAMb$zwz}irx -zX!wk?kjM$}LPbO~eY6GrHgq3+3LfUA3gI%~rk$n`?!sYD+rS_;qBI^wuI2F=&Eqqg -zKZ#gzT=5v+5T#>09_0bg78Wxsln!mjtfOI}0`i5jm|USe?I28BG=q+BI%E~XhZGJm -z0uy8kMtamPJCg3UMcsk^fc*_960XyPXtV}{7jYOuVoMZECxxG)oW}^Fh{o`{Kz%pi -z(8-a;QsnSJ(^?*%(R8jv18USRQiKmg_^T-VXjv#*Mjdg=(fLlpGy?L4vY2h5ltO4& -znpqHM)_&^4)YLGDP_T63^2N5O|3K=)@32L|5^XKBlLlh^X@oTDY@sv^s|7N?0S7CH -z?gkT2i+z>09zKPzfjS%EurI^m*HPa~G;9s^y-351^z#MktO8uZTXc3I?-q6|jeQdi -zgS^X_X8O6DI?q$*KQscX=`vw2r3dd(8piCW^k6q_MYc`e3!F0;V->XP6tv4V0t(Tv -zm};Sv7Apkcw}?FtA6rk8d`h#^K`Qnx19YrKk%;9eJot=e@fpp&hcHx&hyvi;i7iAW -zl;CYawdhV0jmf1B!Za8L2s;3Wt)`IC%s?qhNT2ON)7hIxhg{ZwM5Pd>kC$O?QLhZ3+X-Xjqm;JWHLWaF``<*u~Vhh=whsz8D(z -z6#bk}odtl4M*&-C-YpF640a_PA(o={Bl;OdojKHbf<~a{mhszdVWF_dvG7l$6%bUT -zuP7HG-wT{ml73ViS}Z0+H2*PjF(@gN2a+|Es1?x(r(h~*nWdDzV1R1q5LXDh=o|rJ -zrZ9AjF~H7RCWAVev?I6EfsiQDmn2le6aZ#ai|*V)3#224hDEc;hf^VlV2Y^^B{`-Mm1JIpLrGbpzADe)8 -zcT$}n8w@s|c*}7C1M(sGn{@F;JRwlr9Dqln<~Zr~U$!zs1Gt%4)f^sXKc^CO{*X-5 -z*Wk|S8ThAXS*K@_b?3ratv@$8iz<`PNqLQ<4DhuR9T`#TwB7{@=pMqem_UE)_$=7g -zuH(jLF#-PMt+6y>bQY@vcu;{j^Fs#tAnw(j4i1d9W8dTOd+_}*FSgx03WPjoI$Y#E`VHVIbJj(sur++g%Q9UCu}1=Mq~17-1db?6SU -zEdDJU7c-EUvFU*L|EH|}UDlD{?RYPLD -zj#e>tX-AT3tYUl74+Z){9Bia*iIXUkyGL!vw(OiUd5yI&ZP|OJQ{&W(yj;b|1J+}~ -z>gI>oVBW4_J#Ac8`kT^<_t@I*1uh-0sFbq}&i`tjh$-Y@>oyLly5UrJ!>OvdZuK-# -zU@K?!8sT)edgXnCkE?OGYq<>j4_qZ1IwS7zDsqhr#tcq0U8`HaI*@{i8n_r(C -zjcX-@`tIda7+cr9RLb7fEm!cLbaQd(>zykse06uia8>Z|V5QL@H*~W@%r~48ZaA4! -z>y}Is0dNs~H4vKAbvJhZ1*e!S3&{%WmP~degMx~w>bAP{_G2QbQS4z6PT^y~Df|vNA&%u972#yc -zx+#QvAQKw9kugH0LgsPJ!S%w1{9qTlqfX90o2*rzLUrFYWI5g<@?Sv|`=tS7>(!inP})d8}K0OWfDQ#O_*vODp0VELzuUts9K4 -zOLN@nzuRFxm{odw6m&J`T)E8?Q9prn(QVIF0V4_)C5|C8)rBE@@DaoaPgI3&2$S!nSK -zZgb|6Gii-Xi=Vjv?%wPkK4jOxepo!<>!1@f+dj(5P)o#YrIfJ`R@)5?|LfqattZBoSrKSxL_7 -zG#8u-N>e}UO`Kp2q(0`>aQeQwPq7C=C)mDN%;`86O|?=PlQGwPBqgTHKU3Mnt)F+c -z1txUdr_zef=M&(zJ)a<)_vC&QX_#;l?Nv9qaX;)7r9+K=u*N_q9OBOIWE%YnCf1BF -zAIUi~RGmyxS5+$8-EcF5-Q4mj==0Y*h0pv(ONoEukM?0}R&uAs68Hu`b((YPsaJS2}^*QUPu&&b-H>@l&RlxF)_r^Y_J#hfVYLt|2!?31My^Si=c7b3$A -z8(j5H>y#aMYv#Bg_UF-D)lLz|{4u{;Md~)O(CIFd0g?PuJ6Apsd4xtT5+j%5v|*#o -zKGr3rRXnS%5G0bs!kwc>O`bHk4utRCIohc?gt+bS5nBDrXAOo2Z;%Kr{^6NB3udJ{ -zW#R2j@m=JSYt}70OON?;y{QBHeCCi}jg!TkA>>dgNe`8tiu?DtcXlpKnfuyg4w;kbs3yWBMTBz!k`+^Qn>~{$|w$ua*|vVK{I{L4%PSU!$*%fr%_?vlz`f -z<@mBY>|7jhjv+fl0IJ~Xlvfzo)LK+>eonxBEDbhX&8*12kQ$t($`W6`5)&mc`**}x -zR0{TSNVkdRu$btp)EWOs)WBFyMOsvD!b~yIqX%h8-=vQJzY@K7D$b%J{EWX6)i_nEuMIfXIg+9r$=BQ^)0^m -zsR3A%xZ68QhkKG`7#+e*En~e8j%qrn7}mzfM`^Q@%=d#=IGcu^H^oR6N+lgemE+yj -zg^`j7QJ)a8=sbE3>BGm+^btPiBVr^WQiuA4SilO38|k! -zoWqqn+dsm=j;e%-Y4oNp?J-H}aQY6%$fDmsAf-COlymRaoPjv4EI6|8zxXp1r -zSfkr;-;pR=e};CwKW1hxSDR_TE36^m^*1vN5wi_5pD`?a+OX0Yf@x>T5oz|HnRWZ? -zsN5WjX`!llbBziFp%jEWpaZejGfRCY)>|`M+cQr3j5+Hw3L@?r67D^k2~gb}huId0|)-@uj?|i!KVstsx+P -z`!2-z@X3T%??PkoCIrL6Yg9V?IyU&X>M|4SGfO`kSvu)OG6H2dMGcW;*JhR;nRFTM -zr?M1ALPm6c*;7e6Dw9*L+7oLsOKYCZhI>So@+hACA_}m!m+d&*HG+!iL9&2@(rL1& -z5t7!b%+gv};}oPq31l3Z((!(>JV26oFtc=$OdcZ1t`LivMvGZXd{X|jTMuNG9#huC -zohC~eDT$IwTHnh&t8qVr^RyQ89^n}mM~x|;+b{0}e7?8#0(uHb(w9B4sW?(cyrU*d -zvWqfHC(EKHNvs8#rBmyX2iQqDY_HGDEIq!Vb$4dz^h3s%o*>{=VTqxVTsCEv{!11G -zqMnsm`dIcr`h)hwooKCRb|wWg3T6||WLeaAc57ND7?{{%Dn`em0U#Y7;vBWlUcW7~ -z?sQ#j^Bv<$Z;7}Gl(Z&fg1d;_bKCgRCP$SickGwrGuv;}#Xi{yyvYU;+_=-8_5?wA!0?`1G>v<-b}+dc#_n -zkihd&;tA>V7w%=h*4X&HL5N_#c5gb#xn(MER=nT-Q}~$LUyn+s*4F(L&ZJ(g{?1u> -ztm>$~sv?OO?uD%OZ5XLqldsnONo4+q%U&m=&&X5Hf_APcHSfqtT0OMa81I%~^jZ;D -z*VfqK6(F&U^sSW(72JGg} -z$bx?gCrYj~9rtdGRZOR1(MA4eWS5-GpM^7tBITpB8J+psZR1K1SpIdK5`<#tDL2Z( -zsl9RU{31ih62r`8hKS{+hTg=_GpLp!`A}OK=+W&mZ+^q=vJn2I+hq~Ln%iYEb~99+ -zd~;h+x$dyR -z)mt-CWhHL)RT*Wvny^&TGz}~h@i^~TKX89tc?R)$0*v*H;Ue28flh|FE)CyWH%PQ~c1Hc|3_)^3*m7?HfB;Q+ -zwYNNya&f{dz2z~`TxwW&;vkLD2%CGc)?*|?he))`6_XP&YB%+gF=G`=H(?-e?A?k> -z&N0w8_O4eDcknU%dac>VyJqg9t--M;;l2f4{R -z`ITo2lRBCyyeP?l=hU-ng+?Re`Ri;4csgbnkw!spUmP$})YE05nX=+^3Qd#2d -zbg_oB_uUFq=O_@%B_YeDo0rR0E_bplcdlOUa%H)E=5p8keLwR5>Zre#Zk+On^;)|A -zT6*@i^i9{(-BW?UY|A3jV_scv2&g8=5_vU>jIAGs@LgF>r|$7 -zOy}ud^;0D2bf5Z~vP@47lFG|x()lmIE8$*w?-`OP?&(~wd?ycBnrWFxhr!t#8%#o~ -z+nJ|uHZrm;J-jVl0^_nOtLtx1Qd^N*I6p>Y^?4 -z8@2|I=w|N_+5I)3pRQ|$v0fb)HQcUusRWV#epvG|$ZtP(3cxKXB0Y*sP7lva8AY)< -z$!RhedeDLJx6_G?nz+E9#NAP*8r+nKU+FDb@d92F{^P1&Jkce-MmKW}FK&gq@Wr>Q -z&&jI4?1MjKo+P*|5+2Jusd>O78}v9?&QW`Xh-k_90-DAYEzKJ6$Ob)5N#|r>DcouF -z6SQYdPe?^qMqKf~`3V5#pnuG>65P=r{ySl{AqaoZB7U2aYC7b_0$Q6PzE1-n};c^ -z$K{7<_VYvPN72$y3hKH7aAz1?rLJC;3W(Y@B6 -z)6RL|yS_cGEkM#l%e1zqHRvQYv{>$98ZP54Q^0G{&?SVfEfn{of#?#LKcfSRuVcy5 -z3Psnswls=%rcWzevBO02Iz9OD07s8rW9XX37SQBMrr9s+cXvFs1i -zHWw-mY!}})c++KNp`y%vMQm2q;wr6ir@%O8hfE#s -zihl4P?Q~5Vf4}A1@z;H7jCDoy2YH|=F(cJ -zw)vFLc_%aiHh2*y@On@wFH1RNryDJ|9WlJ0VEN9{E+8UN0h+^iAQl5633 -zNRs!JTr-?|gpjjST@UY({fdK*x{P(?A)5Q1svsuz|M2zZaZOz7A8>Xg8IZ7qC7?`j -zP(fQjTbF7JxS$}tq7_@KEuo#LwYK6?KtV%L1f?yDRdKmDNjtJAwuR$k{9@Dj364g;8ZkTKjvzU{QGC{OeT`qUS3_L^y2pzfcIVmn -zOAEjpu!z>je<-L`l>2Wg@ZVIc4lk)JkuqC?8l|?y8z#z)6Fvu4jo|HtA8tCdpPznph8z*75%YrX -zD;eucxIYM|{Z<(BpHqa6Dah&9@Ei6_8R}J5vVt2p+NgX2_?zMB+S3*Dsqvfoh^a(0}1}0_STE^ -z3kBt;p;koFz7b3kcyHZ+h@@jG!-rPJXu`jCBBmw)mrf}z+ev~(3FWbEyvMd3q604rsld{Oh`S&I -zB)CSJ6QJHvja4EWQMn{nl{7`SdCum^8@K$U>1p-&PBZih&+}AW!LAa^u9D#c_;Wn_ -z+*PY_SHph8AqWeZ=dL0Ndyf1*^8X4dN{Rb|AsW@9*p@6uBd*mu#TXNjxDtn%O -z_C8fKUh~tx1V0rDw$2hj-;2&D+~cp<<2T@{uM}2?X*-G~Z9=*V{qTtXq`lXPDQtS$ -zhQE?REsU-wN&C|lBy6DB5-1Gm;|gQ8H1%ghp~%SGana*RMpTJ9*d7=CkEeC)78S%7 -z(yu?uH@s=oWuc;={`bQ2Jpb{E?ZwD5QJCkCv71ASXNm6VEx7l`L;?A -zIDerR;p@UbrNhI23y(OfFEoTdPjAvZ8xM~R<+q-ug=Pr7;Qy6Y#Npq>#{3e?PUn687H^C$AVrh3S;^}u*;e=^@f!+%EvnKN>8Z$& -zH5su9?S`LL2y|l6&@p(EG32SNk?8So73w4M{x3$C{Oo!^ns0a6ZWM<%&RGi8>#{hv -zS+DS#yD^8Ghn2fpP%4o>&Ijm&cTsRGWq6TzYY_)fDzq<5w5td8E{0}bp -zv(wCZMx*FQp_94I_^a`U# -zEEHSrkZ~~{;{ryX)}j#HF1w7HCa4sVI4&P?l;|h3K%sN%;$Lz_MP(Q>m{TqsYnR~L -z&bbJHo>_x*q2$@o|GZjlvli>WiomyO+}~<6-o;9cZ)I1IZx!KNxfb#%)bI5k{Arh& -zd=Y4eaI-MnN{>r>BP$-fA{m%#9`QMaXBHJ^%fcv6T5s6;rl3Vpik;c?YoTS?`m(4> -z*mfR^jjdI}#PGGk(BIAFa{jo>TJFuBbeYpcgsl$w(Mb8hSZlcql1%(p*m?rP-E^`L -z??Kd{2eZ+t{vT#87x8wIkre741N;89uVUYd^0md3$9TiC#suh7t2GUEF=g-%A0p6a -zE0?W~S+?x%#tLVygskLuMv`;L6~S&vzNjTGuWu@xeTyh*s!;nuafKGH&l0shR$KfG -zs>4IDCnVBJ`y4u@*38_V;CBCARvZJq8;*(kIm(%@$IP60_r%VQt0B!>4+*R^GC3dZ$q7*r>39D`L)<$M(kvCxafXCy!_ZNZ -zs8Jai83nHvQ-M@yXmojM@tA*!V&a-c%65!WRO9<=1_*YUQi=s0@$bE%&W@@oE3?d; -zX}PvS{@&u5nkM6@vg1NQnXV~{F^)QOV4GWi1N0s}PFU|TUHL}Bo`4alqSuu<36?R% -z;n_yV7C}S1;ey2OCxE&`i}IwnA^>bb0rWPcDRZ*7{Jcl?2KXtWV%qqXF(=_*tUlY= -zobbAGPpF2HO{2#OYb1?g@NeFGFJ*G4v`w5c=SKSl_tf|!^LDrmQql0vg>Tr6B{(fi)57jCw2RqCxfgrEws%8ZTOPG@}X -zW2`EGa@V-nDthXD7m15cNPvz=LVTGiA*Q6DmM)1^y(?)HHY-~cUtcDLu%D@sy;0dN -zuMuvAj#p?KeM|8mHeM6neYokMvKQMhb+J@m*HkM+Sq(c1x=$-lDyxL>ViJBPW9CTr -zLWOUr7r&vLD5<4WWz06Lw28=6SGZxx6mIqh+=dwf+@>&~EHwZuDO6Jt6|jsgG||L*>#>2!p23_XZsY -z#gIOZ5gbRq;k=Qa4z&_)RUaj*m%-ZG3wP>eGZftTJ8GS7>P-}@X7U25zD6afM9vN) -z)oHu|8{YXR)L4x>9L61#H#Lw~60=RG2M{MKu*cy`1mn<)j2TkNFYjWq1CKychs_}+ -zoE(6Q&j6&3xI=l`rGlx)`}ZPKUIEd5)H4GA@=Y750fy;hJt9~tFx9wQfsl3tuoJo_ -z%ELGL=nYN!DTOUx)C-wgfi0s|^ZqP_&;=_Hl)%l#FS&n0_oRQ^6bP{Qii^VBZjM>D -zMhBut#qPzs1Go>P_1Cedt+YNABujrEDyuBQC4?MAA` -z(c}$&0?aN$N-&^ASfVV6ElDUz;~5kJe*+WcrLPgiLjGelS%}ioH&#dms;Wm=(4{c^ -zMM-40$Ttgxe2zMO9u=nzNVLPg7xU>LN&(EeydbK`Z1X7*vVZtd;u(v}Dhw4`|9j`N}@D31~MF3cmSG2)zjlib#@-7>;HyKJWYF#zre())l7prjqk -zl`|KH*UO-+pq>NGsCGq#&8O_Rv5_*i7hF7ikkU>08A_Tc{s&^ZZDT5s5@@)D+oT5D -zn#6ZI=>UsQLFothr&(&55JMA&!Q2NGU-UZWJNusc@hSCe>>gtTviNYqjiR9wz3lVw -zF;PM4WbBT|!dj)=o4m0h0}r1Jf}`jyon**;UoW!s>G_3jn^^)@r%vyakxeMN@}>O; -zmBhAHv8hBz3T?7@yq`pkfrfT1JyE`Fb$Inqf3!OZTi`cgb#q9GcWT_?R+!2!R&E!9 -z!kYI6*3#;sKI){IGc)jq#LPx_)$Y+=i>fc_nG94-&d?POu|YH~?qpDUqlejZyK0_? -z>9n5c%y674eUr=@F*tp%l<|9jp&i02LS~6q*QV7=xpO_(mkGz(vo96m=^pIfih~Kw -zX#z!j6KS$ny@Ea4)0j}bN4!*?s!$)+rTnsNPZL#JJ$xvi+B-KyMn(?vRj4Cku)5vx -z&fp%g-Wf5N8i0ef;hn*n2*~M@0)D8MP|+;^fQqioq9)W!xQt*D!(SsYS~v}o@E-)b -z|C%MVUJAS2&Ihe>DjH+#t>9r=5<^X37eMkug3SBD=rL$93li$SxZbZmuaTIq0N%b`Z3i=s>$+#SPgQKj$r)2>-*)nq#~AE^(c -z{h1$GGqX=bpQ9v+nc4g+y;e=mMESmGyXK<eMZ| -z86EI6x-C*vbRKpKk4V^T97@bE3eiC?XUG$AyQE+9s$hYh>f6L`D#Bb_B9@(Y -zx%+;lIx+L3<0mjh37sO?0FM!rSF1QzxLZbfP_hx?vhKU#H;or!57OcL%HSjZqQ}|Z -z_0FF?Hn|=uXFWIi;$@8&{FHw}xUYvz5DZTCi7yT}%=@H9*%-@mgTQ$3ejOnIHDXo9};vcjhBhH{9i%8DmlY(oLtrK@uqV0KHYJAwU|@;G1d;}eCG0@#5_!Z~f^LXk -zF>M$yZM(8%dxhs(%4=Jts3<@%f@E-d2p+`7d}8Qu85)X}(9QjpKGS0C?w%|d7>^y1 -z3VT3x9G=wOU5<|yw3`&|-R(Zt)FH*8RB*#Tn+VlcWrA-EMO9g`4<@+2Jsh@bRcnt( -z!drZp57p{Xk&{HwB~)ie>r&tc+Hvewx)-dyb;gFe)c@)Ir?6g8EHqEkyXmp4E$d_S -zcV{_dyCsPATDTRi8#{Ei+RUun1j_UojMyu -zYmzE{BpV0Tu+Xs^t&>bQHiTD{)m0fi1x4P1sd0kiBcHlAk_{6SMF?briF+eihyaq8 -zBBk49t9j3z7kwid;;4EVi6i_!!=Qr?L6(L~yU2OlbfzRlFBo4JrXE!kY -zk3@CJ)VhIr+x?O{4QJt}Hn4B?KLLPjI=+Ej;C`~qD}XTs+Xi<2({hN_p@zc0p;a{J*mOWcl{ABBI3&5*CC?Sg*CIZ8S={t7;)Yq;l>BHjHiML -zjpkYU5T#+=aTxXexuN7NqIOaf&aPQ^Wev%Fl0@u9ME1HXACZV=5^)$2E7o0EP9jc_ -zh&QRn)W_I+2Q$aX;}YsIigZ7JOP<%@3s4++f9xsWg@bcWF*AhND}K()w0Z*!Wv8n^ -zhr|H|&W^%`8%{zadnv_)QnLf=#TG8xuYWJAm%(ahX9!ML3M`+UA^9XAPW9lfg}@5a -zL^+)HOj*ZHR_Q{bQaz{vo3mr4eV#5R=J$s57Qxd*gkQY}AJL2Fp#;+&O)evFq11au -z0r7Pe9+?%@iq~g0C&afUWPGK}dl>$?5wq2KYJ0}c{rz0~#8<(-CtQ2St76BgFYKwp -zS*e1i<^+2xn%dwFh%1r>5Qg;2Mk~V^7>0$TlJ(xkgvdb>eB8nsmx_S$S|3#(B$VRH-y*<7`>RjYK^#%{JWdpXNB&NLZ$+o4Iof7niq5sNUcVm$y3^k}xXgDV_G=H&tfVg6S2&d7q?C^>KLIi+fLl^;lcDOOy -zfUaXlljFGuKgva|;in9;pA*?z8vg%lV!sKQH;pz=CTi(8+mKqgs%0K})J8njzeg0I1KItzYF9js!n7$0Fd*q7 -z%$+2hMy$qM&U$N>HfWYMc$PM#ylAbdYAqx$GhkA}->wa?YgJY)taFCJ4jZ=mzj-Y6 -zFxzHYZ5ejkGHRmp3&9f)#t-%0$sC^THL8>74- -zR&9h?%RtAUAUN)M5L)#P^!nY-*O|{%lsJBWlM1#7U=R`Bv_Tv7ctFvZ+lz^g6H+Ku -z&+JFRAjl^Xb1d)};9;g|{#B0|pp#ln*ptdL9kJKb9JEINdxX9wnd34@X|ai87le{lIN9ezf -z=;(+=!*qhc7XdhPvYDFWmCX<{*V90LAScoSOw{mA -z)Oz;T(_|zs(s`_R>*rr}CnI!FTCyJe?(k)j7~^YD&oLPPva<*$V+o`d`nN^Vzll_BH7OIXhdL -zD(7C4w#sq%lk&B|=inFzsGPI*3}z{SaXKPoZTe -z(w#M}<<13|zAkjX3F93Brh|pB&`K;JGmZcp>*mW$4TYAfe5!PU7k?(eY+S|xqBoC+ -zOS>-%-Fi(UQrYhVO#2Eg`wFRD6Gn311(@a+T0ehGjc^!`@vSI?N$kZ!#J$u@MKP~n -z*wbiUA@i?pTrJExXn{{;PNA|7m=CT)Ep6F#w?h5m+aM*P -zSfb3vE(NNL{g6P*vS8?*Ux5qr5gwTw@?tf40so&wX#b0gTihKfhit4!C}exVylRlQhhT2lyhJ2Gaqf`Z(_p~K)Nn_JjR -zj=J?iA|A0CsC`X`9sI8>85_1IcGUC{&z+(EB@Lwzibo$K(}uOoJH4uVj{$bWnmW#xd-+edg+7ca;=shBEo5%oCd(3M$5{??tK>Aam(Cr*_w5Ro%JAMZYs*1kG7Fr_geuLU@%L;prm -zw!(GR_wVptAcI&h5JlJne!C5(2^3mygBF4ZmfIuc(5n0O_7m|fNxO8oa9P9MT4hGl -zJ+tu)^O<}86P^yG=5B>dgqA_VIpRMGAcrhgikngH0yOb(%Zt0T~`XZ5v2knkQ~@Pk4X -zt-|QDeD0ri!p}AaOJ^i4m?kpEp7Dh)_ -z4x|2a%@0m9iU7F(-9T*)uWMq2Zt4YVCLf|XwGL9BtA^w7p?TE+ERcI`!J77Ep5_$z_rfr%2N=!iFh<7TjizRRzX0}0EI?zPha;H{rH -zERqcopnQ;HsSd3_awzynm=OjSO$>#1Y{yg{5HFpIl*1iYGOpE&+r6IvG?%pNCZ&k^ -z-q=ia%9?B$O~24h-_aLax?7YE)1N0s%CElUfw1~;(IeNPuIsX0mvDqUB7UF{xJ(aG -z)X<$ElSmZ~XhnDfo2i~-p#4*HieiJ-KfSouN(WTXN_?}TDy1&vuvHsi*7`v@kUs7Y -z(igV)HkqCFP0?MTQd;a<-)aKKd@+F@cUz-Xq0<-JGPMEI&n9a9pGc``gd_ZY=&EUFCF%qDq7t-9kpvyt -z(ThAs#5Gdp=ipkf=!brG+ertarz!EZ-61`+A71bI8)@|usrl5CnRUi-V7OQR;{oCy -zNEbdY=^Y{*=g!v++6j^MQtlmeH6BuIOlL4MHY*{;Fz#!b*)JzbE#- -z$^2blrvp2$)Y5I*s4}49Ss<(Iru08+K< -z2Hv^jVVs)!8`2bM`D~8o -zJ&Gxl5UJbDjFn|Mg$36NDvYi(*>3Q%&ICqDOqOKX^0H0Y9ofoakA|uk@J#f&@WmcC -z{iwAaaZ1~kHPrVtRE(>McmVjpB>zq7Qw6nhIhYCwOjg5CUr3rZxPcpb8O0 -zU~_{O$#Zu(NOpApr9gLq6ypvchUcHg -zuzv+wNDO&(I?&-ENcET-=KEKH9?uHA>28m14dun9_qXkxHB=ZglZ4S=kTY|h1;e;k -zNf3t!CYA)=L_1(-FcG=v29*+Qpny9O~@{Q#6?9-L5T|HR48uIDA=FWaES`# -z{73-p;%1WoDrejQRL;#JF{JV|5_l82+<_&Caew0)#E<~mMR9*4X;2__geeZBmNY|B -zL8$pXCiYz%lTs!t=A=U_pB$W`i8z7shMfrVfCWFCWW~3!!|`RhNtgJbAgXV~5@w}G -z|3Qn*?LTNyA)%-F1wm94C=}d4XfgIc_meFUSZ~Ri&^)$=M7$K5$5kW1Is?hcjTZ$` -zJCFyrI^@Ca5T1lV1kuXi_8yW5f~eIX+;AI@Cl~T@-$4>J;SoRKet)aEtE8Drn9aBA -z#Um8lr?*L4d8B)nC^STPP;%xy8AAEDL1-?c_^QVs3TJz?(WN6hmDkLEQ(k3{l>lRPU?mF-g8dNsBv`9kZ1kKNvfl#s^;W+PiOO)P33g -zY3!lK-a|t5;cL6E6FC~HC5j;Nzjho_Ciw>cPo5g&F`6pIazStF(4W1axBQ|v{h}v_KrpXC_d}Da -z^w(;$gjzjSA~<07g-uwz3kfgC*!UdSP~;dkbf1JuhEgdtZ|F#f>R`6bjpI+_d;idd -z_?LgDr#4wy^wj$&g$K2@%Bn0BbQU6!w{ADtV}^@lFQ6YAwKYPV -zpRuo}^&IVZm~g$|LitGx<<*=;OwL|(H7aXkn+_YTms|(w;;aiMsJV>I+Gv)>RSJu+ -z`qPH(Ysb<x8?%~~OaVw&VAr<`Ue_Y&tk2v@W0CarSR|eG+0&%jW+AsW -zc!50TU7C^?r%f~EST;@Gl-($aIc(q#J&Ae2&@PX8g}o;ouO@7e8op4sIYZc -zZSx5h7;N1o2YiB{s*(AVF!Oj!s|<90%87az_FraMFuORI0&c)=imAEA#rn?rVn(99 -zdxjk{QyF*UNiF1cDZ058f3#iGd@&=lU5Nr5?E@XHTkKzKH;mk))|ALnbSEmh7hR*g -z(mv2n^h)?ZKkOPMj~dx5vThN;R3*RfNs5N-gSW0M(i>fL%DIQga&7mynrqfCW-!0H -zm`i_*lv|BCCas=5_5?nUi6(NMz|+fV66yq|eIQm=a_?)aq>kr5*@ -z)#=oLY1c|lB>v#C4$;+|z+hzeJgGea-hgK4XTOsv;C10d^MNaK);!oY>_q#(7>#3z -zYxoIDq?s>pE)YbwM7Q_u?ine&suCT4O)&OA@wVu9qKlV|T0FNTY0YeKf`}BHW8C_jS9Z%$&Aaj)&04zXJ6PDJ$cL)Yw`K$IY8Z>*Nx?5LbUw& -z1y@b=_A%ANz45&Y%^qKo>|X%?V-?w0lig4^sc~pSUDJpq2ckU9r!KmIH~j`D$oUCp -zp2L3>{JIW)C1EFk{P!pkP^o&2A$-WcK?~Uc`Em0F8x|{h>Q?CvR`o!aETR0#iBiHI -z_`?8E+IF~PXeI{)Y#xL7*Mdr62!khXK8dj`=!}vh7M2_@%9~&ABB#k)WP!d~y2W&S -z9k+s*LSG!f75uS4zS+cNqvdX5ER<;sbO+{hHg~|bA~)2FSNDgP_1N!6= -zHiXbeS%ZC$!j33rpFUxy`V;$vnOY%g_{M|){c1zTZL$yG=kUC*Z=w?DmE{?#~A>9rC -zlBhNh1T%7~&P2fA}`!2(%;fJ!mj)|pQ!nSmjo5k;+1qSeXIba(D;e1a2z -zjz*dt>Q7M60CUI?iJl%o)$(rx&fVNfKF6EP8{+0B&F7bB*9h%tu#dK58PSejw(|{ZmN6c -zu*$D}BYo*ejgKyFv}y1GU-$YG>j&y}Zlasn>o1cDOu_3c*YS?2^T$473g~~w -z*7}>l&Km-DUgF-OqtFU4gTd@+yO~oZH`Bwp^GYW@v-mZuzK-FewXU}0!|uFLSWC36&-S&^~<%=!(y5w -zdN?k!!+(kdJRuC9lU}MzzavpkchNuHdc1;lmIFD|WA-2ZEl~uhoc{%P0o93Rd%p79f(GKDhG6iWhkftN6F%5}zX;ZU9%0$H -zHZ+ud60|@*QHG8{9~_a^h`F>UWOgT2f=pyifZI>W?T|Ozffp3AU1wk%GbiD>4hV@4 -zWvG*m#j(F-l;+D~{FE2|zydgyqChL4defsEcD;Ygd)#e^FBmnJLdK3E;g1=968?=q -zND_g=2t1PwPMOyPxvqu -z!D?Od4dXAd^V(CrtN$ZZw=eM0-M;`_4|5D`1Aq3g6g)yHe5p4hRX!VFe<}Z(N63}W^ -zt&gTW9Q!kg5W6FMHRVI<#R@C`eULsX0{yBL@E-?I(h7}#!Q{12ir>~KH9KUp-YaK= -zAYbK$_p3*Vc=3H+=ASBO#Qv&e1*7j(-H=m(ti->xju+gAewzBAl8_gpZ`P8(ED7_(SU9E5%@NtUt -zH>GCLG&l@z4CK}an01A1!1g&tuA>xZzAoH#i7gE1LE+wp$b56Azq;6<%XfZQ<@|7m -z^TVB7={+jA^c`!Y9wyj#ROa>V%H1`GdVi#RCNY6T<}u|l^_+{67X9KX9YUDBr9&x0 -zST~xNKT_V&4+$juv)SQc6R*{rdeN{@IDLS6)K`^4N<2>L -zejj@)|1X_A`bItw)z|VDi)I}gwitBr+%2#t(N8nLq;^Ms!LRw5-wqoK=2BF|!q4sx -zhf8YG-q_3e!(gQQsjfzWfXNC#=YSh}rq^sAVL6{~`zimH$k7rH(^8Z~HOkv?^pUE~ -zTT!Wc=xILsQ1zbNdh~(nlH7juiK^7warCjO)!TWrPxXo1bTXg6doSo* -zI)8dwVEQ(nyK|4Ndki+PBD96cR}N1z+48{(rZ%75Y$(w(dmo$rlTYXmhePCO{f()6 -z%>4TvK_i3=ge>K+7oA+qW7AjveZI+66LU=a9@p6OdVm))4k-le5`!Uyn%(qLrR-;y -z6A!njI~@}%frJ9gf%=|0r+=jqI3qG&aYu~C0E!h`yS5@ -zJeN|HXC_p#hTfu6@X5g!^iuE0n(>a<1zu6~Z9m_a@-G&Roz>DfT?E6ZEyq=F{`|;l62N*>oPYW}1%FCm<&@m7K}Y1Kiyv{5o>=DPpM5otu;)SO -zL^=7%P6AEmt_1P%PmJLinP!2E7MKR+xyxXl^Uo{EyJ}oqsq+;Xb;^3#5sy}ByM)^k -z#2@Qwm1BjZ20JSyno4QMH?azXASfQJ;Ck>eSar1gZx`A68lQVu6_-t<3a&n(*@GJ -zMyQqMYqm^h8gQelT)Q#k7;^~0}{>(6`>JLX@|9h`Ug -ze1AXic4sTW27eRFH#r;r-DLctsrp|+#bN&1<60HlBX6095p)kc0^IfzKKK>roI&ic -zT&n~~6%=9!-{qH&7#|4vx)70j8&lyHqVQ&F7hF4nNb0}W@l){ZyYMrw?UI7ad1Vz% -z+n`Aw4QPF6O?l^Su#uY!mUWMgcY#WhoH;YPqhqvjp9JRE@KbKI!%9v}$LLy%zDwEH -zMP%s83#JR+wF-FPk@KB?wKmj)+gUUNw<~DGHPSpebJg7zj`v#%DHY56r9w)@|K>MN -znM*H@oep0QzMYmfQ%9e&`N~OQXrKPu2z`4*cUO3WwduiG>m`Z4E8==u!{LNZ8&RC- -z1XuY*jB8jMVAcVXjCr^gQvtQE?Lr0Px%{q}veC;LMrUMtA@@(YE;J|(lM -zt47Ig{hhN40OEm0mYdj&_0XM_Zk6J*x<0Lo)pm`A0s8oOQkax6Qk&qEHWYp -zJqO~2&FyMVgSLDq!E0KznK)%rPaN7zw^x{5PxzPdL325BeVv_o50cNcdrRfq?0ZjS -zw><3h>sER$R`*&VbZeTL?wvVT1{0*m1ug5r5yzrWv0lN4jn+kkkWFPltHjt^Y1b}bmUfx7^Qb{iZ3ez0 -z&z1K8mMpXx2JO5a?E)OHvVYHelrOhy{7om?R2YlT7_*$#+Q?7ubi9?N%bYxR{}}(zwV&_ -z(FrXtQNcKS>HMU2IypVm1u)<^tRy>Krh}`j)nt1(gL%nIZ(7!^GHc%i3S!MF_ItmI -z>P0Pk5*;41-rbRSY|&JOK=)Blg~P)@znOS)k!tJ`bkVL&l-jy@2UwAI`c2i0H}6M@ -zik1kG2N}#m^vu-Y5pwI0byVJ&!1j-UhlEZgdb4toF47poZ43n+O*}`nHAbuv$U^K^4i-V -z;XNv}ZyS#ONg#)tGc&bpk2XE;zW$Fwr}kYdy>Q=}5q+%AFZ;dl&_15^OZ;*j9I#sH -z;LyJK!iS7)THxh0P?Bg(NFlTjU?IK-71`a`w -z?6wK)LM(awCw}%*_tIyK#!~xs;@ -zW-n^*iM^t!j|5IKZm#XVRYAYiL+hvk#B!5%kxJ^P3wRkn)Iq;}bDouc+ctkq!{jDi -zL|D?-q9V1>baSO!2vqNv9x@fRmAZ-qf-{qwuFPK3LoZ5tS=cg=VgO=z5kfzf+s79t -zn(24bbH*AMR#trbXZqr6x`=^|x9{(igU3ltY-%mEYZp1`MY=%>n8p$_PoTKL7K_+# -z{i!gk#vh4b5bR7#*@%$8R$sORSkA-|QO)HAW7afYd0^_awONc^%Ii8HV7Pzb^>FF$!53vY3N0uPU(z9ZPLCx{E}6e -z%nR^PeFTC#<|98RJr7UU4gukYJ9qaMHLypU0sLIBfzAX<4r0U#?K|5Bz}pw&*JY*W -zHyi?g(oDb8(C{w%;_p;28C2&KM^*0+QjHqaqw#mrNoOCe?gkSU-o!YUfnKC2PHzku -z5!HC5xg0OsOiu0Fowf!Tvuh!J@T_al6IX7oMrU*YGPz1%GOxCnS5v##azEgV05%~; -zOyAAr%KV%f|M{+@=RFeXY4OR1IwW7%jkJuALSxubYf_E+zT -zrfRV8ks)r2PhagzeMr3~n~vqks(p*!P1e>2skRqa!_aRpOy$M79NNWZ?POB>6f-@A -zndi6tii2M4q^CUaQg=DEQ>tqRh9;}OIp}quP8mAmq!-p##iu1+KaB;{nsgfS>s~#E -zPOG3NS0zs!|4Piffy70O -z3-iN4Co5Xgzi!^;MZITPwc1Y4H>8>MjrC?0+j~ -zpna-wAG1_>UJiQtj>L?1XZnIKm_-kb9TMle`l+A%Nv)^;;k3;J$|t2NLvLATw7^xf -zUO&3TyVfhEE2Uj`@K<0=?6eY!2I)J$>~3jFpYWxFe$7e0TI`inGjzK*$d07pgQ*Dy -z`ZX#_|8bLkU!mItOm7G{eN!u<)~Ty1-dmT_Wv2g@(q6s0`S`v+t#rC7`jFY^Vt&45 -zHr-_h+@qrS7&1B(?DKx?G;FCexF@%aO?R#I4C9%5Si`A{OcMAHc5S-s;-6+)4!76^ -z_N7ej+L3sCK8au#xsJmVQkY}$j-uCLDV^&H1hCYqg@npkp6E=Q%H0m6#NZ=ZwMlEJqAd)F>2pXLsUBt3#@fp>?P;&-ChbC^otOS64SNA;Mq9O`sAPM& -z&Yt!j+ZkxpjxlRrbQkvqNt)JQ95eB;JLx#n{ak9Vy?ptz)GwK}L^`aA2^HlA*`YrPU1$@SI&xvL)EP+d=M$p( -z!RP$Xfwfr&wVTgk{~~lM@#t$J86&-)*_6hh+uH~CtBB=p{ -z>%e%SVz1@vT-%XcSg?U6@(uyl76>h8rM+BaPm_&}1+7hN*mx}RF-=@Swl5~p59quQ -zKHSX@EfAQ#%(dKF(JF#1WlMc-E!`?I?aO7p53=mbt&PRQ-dtEcu1$anBQ;ed0Dc#^ -z5UQ-X(6lGl61{?J39>}3s1+jN=ehjppjzeAgjOUB!9+h*XzE&J*`3Rr3bG7FHTiPW -z&Rq6n&{Fx+gia(3qo|J7Lep=nET83aCxR?na$7|Uwk+3{m0K@m7WhCFADg#Q(JD$*=n-LaD${sR4|riAJtbSgKO{_>LsE!bhNNyKh++=8TrPLXQy<(WZHoE1&-!F9&OTOIor+oi3Py8yj -zAt*td8|N=Hj+(T`@5OqlMU>kr<_`oJBm|sL14G_wqUR?9fNt8zHHUUO>rS#DpbA>m5OJKv4$+*-) -z-EN$aJqooR9q2^FQ5+YxO}2~=#H0}U*%tK!g_Q< -zPWvoB4wIV6{8XaCJlqE*zRE-O+Bb($LG77I#RIGIS4FvayS&=(M{HlmNPFEF+v9R# -z{DRvSjP)Xl{2*re{e9yj$J3F?gCggtVT-+4QE}>_YO%t4>VXO#mrwPnMkpvLx|%r+ -z8j!mPUOpvCN|%%d7>8DZp1mknX^8eAUSP{*z6(egmQDSHg(38Xx#}SRuU(7W8?aFA -zo2?#Lt)?OmdkXeOK%RRVQqZTktDmVr=A!&-`IV@Z5K1loK=0s&BjEU;z!bTH-Fz1S -zA_cphm;qJryMlX6f`VBjVb7uhQ|oeqg`&dvE56_@5MyMF9jz?-(Zzg2?&sXLo;06d -zi%ride=P^BPtT`z@6hsWprJTiqS)-9@>g>VpI@`g$iX>LQ3A6nWDFD}=M)F&g5cT^ -zOx-rmS`i>w8fab+Od=!d*L$WDattDI%c9L@;Fpif$iIG!pL}*g&Jp4EsAhm$x9t<2 -zj7q6HzvB<92>Sl&!`4JGzbwd(2hCUWg_tMLc(mL&)kwK&ytH1(Ja7T;Y_csw7?qO<1$Y| -zZz9(HvG-k5ANBLRW2I<}hP>;SH|&O`W1X*JKEQ~~dtG3~?-rRJXGaE05|fc8E?JbA -zEKE!mPlpS81BoHncgO=H6MZ6my>xN%@;|bRJzIvs{+Xv~UKULMxaX@$Fs^)}h|bI+ -zf<|U@AY=i3+$Vv|sJrZTtpAJ-itM4M!HQ00Ha_4EJm6gcwFU58FFqpV&B4oUp*TM! -z7ME`?NNN)mJuldGkU;XOP;Q^wXEL>q`KmuKfx5-|kl<-5l->F?05IM_5(H|3tL+b5 -zX7;-Swy{~r;ng300by=Le>g6So$d~ep!k;p2$%0jHVm+K4a_7c@7HBi=n*LoL^~1w -zwXpx}4?ISGa;`%hg8a~@L4I-o_87?siX9V>qRZa;A83?m(tC)zcrw^0vLne{fJhd) -zjyTvaal5!C9qd5jdIV*!PYi{=g -z0z+hu2bn6f`Pc5cVF_CUO`m4-v)vDz=rMaEU|$_2Nt%Tzj)EpI>jTJnv&+=Iic7ou -z-xm~T=zmc!%6r5o1)0`mbAP)FUW9_53It$iG~6nnd<9b$Xo|??lJ5vAo=7V -z)1WMF#$8ZCZ}JymEIVnIFRF{?>OIM?VfNt0@Af1W|3-;snaUF0&8DIc`r9RfQou<$ -zhs&g1&P}_k*37NAfmP^D_9qwTbC+9%p@K$J%uUqI^tw%NaBr#HYxnE3veemQGjRa( -ztrBgXls??IM${aap6t8zvakB`%&*JLI|UcwmenBBoT^$75M&I-0$A}zwa)(P`&#_%Kv$B{G7`j`_li3o%)^YLY$RU;cgt{ -zUr$3s{)I8c*l1X)iI4{K;Aa*_kG9Oid`#CNJZB -z0{~uq#NhTG^@vjj0KEE$RRYb8boVFu5~EFA=RKgMUs={J;lD!(trFyXX&G~Ylt%d8 -z0)Dq27;u0CY%@3pScfTNvL`Lm&4wt>(A-jzb?R{VM@I8`Q3FxwonlR%?AGm_>b@pn -zRp5#SRGb$ZgsP^g%H%}DbjtjLNMD>_nC@jh^}ajc?Y<6Vc7<4f5SbMp+7%z@AyBCc -z$JMS_YE``7ABNpQV>QhVc~o0q|3PG>Rye5T|Dyn#Wv7{tN9puhKqZIW$z3}}`#!Tw?Fs*kWjnS9lnn2SE~H+;Igz#nS5I7er__8^NA^$c8Jab8sV -z54e!u%x1hx!<=Nr$-C}gMsY&sbb(>I(A{(W-6;mESI+GAJt9YWbw1$U!tp$yb{Ta9 -z=2wSOf7$%4)X3duKpta^kVOaYyg6_XntU1STHqj62#X1J`}Lv_BqZTqnljE7$Ii8(`z?F-%50U-l=hU~rpBhKQs8ZEr&PnyQ!hNkfs -z{M~4B+(Inxei)j^G?CW?f&L9ifB=KQA$Q^bA*CXQKjIH-=8sU_5q~H+^5M`VMBj%} -zUd*i?wkCkTdY$mTC%n&svs%X!?2#Qb?B#2b$)1rRUL)Qa;Sb*op?=h{sXJXWMosvw -zsLI8(4RD;goxY$m@mOEn(Y~oBH@)cIHK2*W&43CT+4TTuoJxW)u;7BiV~RU$d-+4k -zi%kwNd>#(dd*m(Jl1j(`1Z~fK2&y!Zgx^H|iF{Ijum!Dk2bYjo)?Mln3bFhYl3)#q -zWu68R%TDgkhO<9Ekpy8u0ZN$eAdDwaVcY}~8;xvUT*9+p81oVd!d?Tah<_Fgmj2VO2r>YaA3U@G!d%>;xLe_fNt3vOCP$ENs5nznb -zQ7c1V%v!&SO0HGfiKfZI(DDD(G~omRZjF+KP(O0N3=5m();yULaYV}A@`q+Dz=wR7 -zzkw)YEZlx2ZN;j3@if3H!5r8RkP7Ai#r?+}$a{?b`iDPMX>o?*zwn27w9g#n=T}sC -z3KD)?u`u{AtrPwi&6mjWh^H66#OZ-ba6H{jRo -z<2%HX;2oUijQ46y{I$T+Qv{UBg%b{llnV|9lPzD)bp|da$&Ps35h0lqRZ -zKm{u-spNS9=kMENG^tJY -z|DZbJ0q|{`3UD|DCKe~U1JU?a&;T%#h-oER?hr3OaXB&5!TuAxResYwr1=yVw(=fO -zCwY&ox(@(7sV(3GLuv|G)c_n-;rbiizy~BQg#8XC1p2rRnGdFjA~DIBm}GJ~1|+^U -zIq|J2knK6YTYtTrX?n!{GYEvc2k3fuG6}DP>Vc#pImwzl#hyHMdGcff@(=sRBE -zJ5kPk5fsyy5DrHImnGI2bK+*a*3g(>ossUo&K+CMclA0=0>>8u$2$|9@4O6pC{cIM -zeZB9#_VGjeaC*Eg;6?|OzDHoZKK%J0b}L_cihVNxh9De(f|s(1AhJyRS6V{JA5?$9 -z-%*+@D=op~Y5z(FPh(eF0^ILDWO|gQ`jr-6JUPrd_O`NUqYIsZuMho4=xKwg3`0l% -zO(ERml3p0kdkc?$>#F$nQF>K(XcXfuoN6anR`lCMIeRD&mL(1a9+9-l+r=0FKLDD{ -zdXN*>SAi`dx_4xR56BF}jKORQfs}YU?{`O4C*h?JRDhlb?^?{fafd93I|i6Gt>iwv -zgALGBypsRqj?QloD?=5G^xoxBlsM5WV5Rq3c9`K%{^L8?Pb>dGcaOoD8eCjra?MTu -ztyyWgyp*EY>(IO`xVWOiF+jh`K*}e^(rq -z91z?(Y&RV4mtlj!m2m62v3s!IJbU1(pQZbStaIEhf3i*flR5=?PbSOJkKOd4?j4xM -zX5YLAMhJ%XynEP?+j>kP%Ppp*{7vA -zX33qF@yz>oc3otalF$ctc8zDBC-9R#eq@&^aV4JzV}8uYE;j3qaY`kOf&p#KEDAaf -z9kmc<6xLX#2QYYu=c!O;FwRrV?!eOV;CRf(*$*jV1K6d?r7+q|PFz&RYn511-wm32 -z?`c0`(c4LW-=qLG9_t$ydLuG))%pGY|Mz`AA075u>$$AwzMl20XFXnQJN#sd4+h$WgnxTrQvF=~ -z0_KP2=^gM%+kzFaZsx{1lgRHU3a8`b{Iz%hYwH=@$xPqR1t1Onyw2gLR%lw$aCimh -z`xMLw+fRJWR!W!2aC!vJyi6_1Qq5Da3s`>qeA&W;GIOb^X*>%Bg=<2KGU~|8gd-^- -z+an>*mNnml&%)*o>l~M`ISRZkVA&XHoujOpg9*QGrc2mNg^*liovEyv>B>E`f<3CU -zoQ;IgYJGxgdBQt3T5`YRBmMf$d+q%&Gajp_MoV(csk+ZEh0QJwoBg-2*)KNEexYS{ -z$s!0eFzcoxu47s2r)uB4{_n?e8js -zoyJD$G741ZRmE=Z5x44O^O(kl0j)S(?~)dZxx|DL&{%|Fg$rooVh@fc@Ue{`!&XEv -zP0PxQiG!$A6ef_39$~Kg-t-d#40kLAwyy7$U0>d7_jN7vRl<_2IH -z4uz9H)$rG@i?as^rwd``z0R_9Vlb14+$(9)JmZ`J(Cc!v4^EF6>Vv6szS0c6ASb>F0wQEcDkT -zd6&igcvqI~ub-^r#}0U}RKLzA0RweVTv~N!k`7hl3KOK%N?wf?I9911ub^{@2$2gB -zf**P%QzJ50y4_Byi4oaB_xD`FH?uGFP#Ep()B&*NQwGJdlVI6SzuR-*1ex;K2NmUH -z4x$3Z2NaZlnW|*NS!jX0ub>AXCqF$`m(NYW!u%~pFa3_5uk(YA)+3FQL10>cJ?wEY -z(GFntd4fD~K+vo=He*(2D+f3^qqq|U4}anSMa)mO43wFFZy9N0I2q$zZ2CLjBO;cU -zflV*KMj9V#gxFi!s=ngJFYBRiceSE4S2UQcwwKaByVFnm9jM{5m!Z3C)^S<*D{g{Y -zUpArthRdWB{O(_H(9W;Ct#(sz>J>sxA^WC|b6p`=3dPMj{wa82;TNKY#mD|8Le=JrrS)CXi8W#p5~F+3xtm(7y*_SD`ryF<|-LaJr| -zaB%iku2z&YiYB8%9jH(TD4MhiJmZN=@@h5l2`BFPS-%-Z)3e$KcZ!obk|*297x6Mg -zFV!Wd^$>`?+_mlDb5e+(ySDW^E#dYAB(J(hzTzy`vS2K9w4+g{U0Qq1(yTo?Wmg}8 -zZt7?Oq3xQbRf}z|e#~t`+vz5nb^3x^IUPbUIZ7g^fr+yS(HfY7}Nt*gR#!kJ(NhLf`F*i7m79v*JNt1VSCl~FTY -z*(JIcG_?N!LxIKgZ>6azMf$y$pmr@&E8&;9B*}o^GPR3I4L8l?J1faRpXq=pn+vWB -zzjt-k1^QeEjwM{g+%BOFd0L`_7dCL^%Ue}!E*7VQDmHTf?LFWmoRnO;0CsLQgyR$o -z+)R?WG0EP%%(@D*0mhqf=taDFN#D3`Z_3u9-Dbj2Ks)1MYU73G=eA$nF!@-6euHM^ -z6pzz~7o9p&pM27Dxph+DB3?Zp>sF-UvSrn;?f)UqvUEM+VtPW+c`LH(38nPw%Bpa% -zi@lz&R4))l?yDAM*G$*CnF{Mn7h%^TE_ykisT1B`1S3}pfS7@r!fokQ-yAz_*oW)1 -z%2Ulbb)To;qkelNp4X9|)M<>x+dAY)9c!#B9M~z@Tm50)=qff`>uT|cxD?sG#9-HL -zDQD`B!7=-_OifseN9E+@Q7x_+UUkv0mKiUukA|}WOU2NLO)eGiXlc4wIexjdIy&so -z27!+_5mNk52aRq`v}B?A|WQoyec->h-rYRopdP -z7|}1(&}C%y99g@qx`|O>9}`!TXi1(PH!l(0|J1dUZY?byS3Q7w91uu< -zhJx|x{#&q^veziS{@R8Da}pf-@-JhX#N(|jS9-BK_uaC}xJ77%uohL=p|zyO*J>-j -zS+>&^pQN5=4rBso%99zYGmjr66fCkDFOs*-OcycvoUUHfXuJqDS)=JeDP<|e9bymZ4Htmq6L%l^uLoXo-CO$8`|h{g -zKXFa7-B!k~j5Xqp8#mnUDh(;M)v`jdH0Ccbn -zeqA6@S%U;(kCLru!RGf~W9E1~2%NMC0z=)vj=FePYovRbj=W@6mfC^eu7Ov&;#Jm2 -zPiv${yNv##*qJ_**J#Sx0>?5n{0yJE4@&LO2{A?@g|)ba9lDH3<1rt|^9aj6<}=3; -zWgQz0!BXC1kw^R3E00kun)ji73aT9C?eHM -zK(fBgr|{08@3IEpWguy1xtBkqYk1ch{C -z(TnheE)hHjf>72q{6%V3h0V~F|Dq)CXTxzlrC#>iQW)-BSsMJy(v6zE?&oKfb}U`9 -zB{gEV_7f!&n8b`qC~_nTmid#lTpu*!+3(PH|vyjT66i~2WP#6BdrXLl{7!JDm8>Jtxcg(It< -zY|y4tgCTZ;N?A)WiRFh#A)#Hn#_Q1&8%8jRTIvcJN8{z(Fq&qPO -z7_8R_O1z41E@Y9o9_$jEv=IWk=9dj8h{2L|>redU?lCYJ6ILySzn5p0o?f{Whk&|G -z+$Qf$)O%j+JjHvG^SGjEmZE8rLOnyFeoUdBu24rR)Q>9EQx)nd3iTrjHC(_ytWb|r -zs6!Q9u8JyGm(I@IijwG8T$o2axbp?#GhM)q8ixyg?r<3}y5ghA$MpVaqY{NlJP>-cP -z*Bq{fBMYvn)@JF7yy>H+hQGp<@3e7s1#Dqg%nsY=J9z#v{7%zqTlnSjGF$j}F{QTf -z6Y3IM_`asU*}`keU$BL5jw!Z$8T@$m|HhQzV$ToUi(<0mGRpkY?(FHLJZKKoF`L@xs -zo6NS+lgnA#=#eqGw$X#sIkwT7rfeIo!GsTLT42K?eBmRTGHrDyy2BqCeYqB+IgRXF -zTwF@eFR47gWR2gdH>#B7Mg{lsqGk09g>RQo6ZZK1t>NsF(}xz;H`eR#dEqXS{z~A5 -zua{i?ddW|I**E>>mwvUR@+(A5dc!Y)dv&33dI|0u>AxF-OG*0mfel|SIlWVgdtk_! -zs!$9-jJD!iUH!i(fd_6I4%^7kY@hc9hT)d+8*cM8H-sGpmQ&;5g}Px0y#r&e?#K`N -z_5B);+|d{JYdO-TAKb6~$X)#xL0w0F*Dne3dRf0cFlfXZ{fdGHZR?jG6uYNiPEh#K -ze)AbRo#9iMhm)Ac5`qq%>6buxs^PNWiy1!eBgd|hm7RYd+PQgX=boXRhle)8wJ34$ -zsVs|h?L@1p{ifpN7X4smtSd1aI -zRiU@NbWK{J4!XEfHLoz+PpOmVOtiJM3gPyK(cE(9^oO -zO$%{cMDcHE$7nuY<;JBh5yQLaaX-AJfx{4MCHX4 -zq;9JX6YgtT%ihR{LlxFJ$Li)BNA{^ZJ=MExqUKsV9_b$Qmb&e&)#Z0=yy;i{$To4= -zO{f!caF>vs{fni>f(w2(W-DV>;Wh{yJ%d+C28}H));S?{ -zb3&0pAb-T08!J=U4O*hF1N8}2%M%W?Bc^ZxZf5_PUgi`TB=64vbf>OVsTMBjzI%Ob31Yy -zdQtr5mZb2p^>AbJVIauP?&f2E*>XahqOe*lbrvgIf0NoLv@LFETTJJ- -z`5#Y-R3&2NwN@)%AE{}qcKI<#8Ngzk)e$1G+KEX=&7%c^c(Wggadu2TlC2sK+_ACZ=4&YHW -z;+x)ebWd6JxRLwI(nGtd$9JWNwO2pfo<6RndO}P3_{M5wW4fYb$B>&H?iu&C?Qn~- -zWJTdtgeOc{if@;42!zfe5 -zPVa&!cEt9UYpc@QsO_SQPpa0sl_k2Q9q39_Qr5>kJ3B+W5?!x&dKt@Fl_rnqzp!nf0gf>+7UTVArsQ07~Nk1pRreB+}N -zo6h-aJik;m-#zx%ZEt>IJ)v!E)|R!pc6s<#dH5hv2%uFS-uNi<_+IMn~92T}dXm)SBN8>7m=ML6+y-}*9LV6-64*IOZ*!_lEk;a)tT{9PV=0}=l -zF4^{GJK6~8pqXM@>yBSD#=1nsD!O8ordZds15vYrOtT&|&Kle`>j7kgjFDfF2f`jC#pJ*MedD18`=_;R~NOYF=}D$-r)LU!Kcm!w?ySP -zMitaW6y(Pp14a9)!e*(~RE2UM6~U*6W$~FeRmW%J@x24FOkfx@YCJO~l!+b3q=ztz -z!kFjBc70ybZdrMA8g)Evtuka*W(2-cWlQs>f5&CgPLyIj*l$A -zVLLvVoQzEgtBU0zC|y>OM;eO$_|)Tc^8Ojs-VPSDp3S -zGN#T|ZFO~*!|({>hcF@u<1U3!pEZ&&9#&URIgE%fMueeL5XMsq<8d~Ogwa@Cz2q=- -zn)|FUgkeb-FDZ-$IO}wFj(hg1A`AQvMzv4R-7>XHJt1!i@vk4zf!T{3S9Q#ag5tm9 -z@=pbB36>I~v~dH{vQCGqQ%MH?^5r!Js?nYmNrp28svwO8O4m6ZRBY`>21|N1pZGh2 -z1KZK{Nrr@&2?^>+Wgk3RlBDGm-K#7m75_9?Qmg5??n{pt>OOeXl2L6{hbP2Ls!h`Z -z* -zH{!-rOJoTQ$?Dwig#{a0+`B{K0_w3X%pQ`E9o*>N1LHKJE6X0*>c+NS=09pLo>F|g -zqFP&(&1|Wm*3}-@S_s)N%T!LduyMc1p|AUrYXoGRIq}B*SSZZO -zCw_0)Uy&Vmu~3MSK>i8=FBA$z62 -z{IpOAlEHSFw|#}e03gH*4&K~NTJGaQ^5i--!0W41XRGNp-~PbW$+&^QeOoR5$9I@X -zi$H_CzM|%w`mUOLzmRYBorrKXg;bz~JXam$#h#avCH04Tc(+jaghhj%^M&flS}1%3 -zfc=hy038CNyW+>Xw94xXv7rzao6AO&7;yU$>Sm(-HBjsDQ@js);1G)ny$BG(l`SkO -zUIiRJxqlQ298v@di3)GC7Q;218mud1zxB0LT(iq$6%p4JpXh4gzt~@ObD>KyY^GU- -zVk-0*;34uc@}BVWT{c6ODr{)^(q!VwV(B<-);NP{r!8xoHf|irG%UAV=DF3?P#<8h -z1`Eq!RBFL3v>FRI6Aqch=H7DKLoB>6#C|h<$`^*&%OL{W`)hAnS0iy2{+K7mVk4kaDjy+?e#=coI$T##N%4Q$nn_1$EGn_L -z$+8AB;GpYK!`2qMhK$!jPi2@+Udf2cTkEK&>{%pw0#oj8(iE>Pd0870Hx#Q!f0Iwu -z2mc5u8Ct(qSCOOzW*9KrZNT8dw2YZ(@&Q;m$_$?=GKQpNM&|YT2cIhFoRKFYbTnNanYojf?IRLEG1Ek%G<6PEKgR4j5l08oLe!y+A!2m5><_7 -zIjbI3HYP{Amy}Ev=wj!!J>zcOY$?B``FNzzlT?(WW0KdkuHTdlsTjQu#(y;{V -zq6({SUkwHo1k*Qq`w>M6TY4$sciA_c`afu|_tFq+lQS(y_(vaYl=y}-jS^nzrD2m# -zziua@Envz;31!Yy@m0rIl4!+!G|P+h-<@d1!qZN)5_Y{4ZG*75kH#tZSDa|YLVh33 -zQX;N#qGbrCUfO6cx?Cd3b1YRTr1#QboAgpIjhZ7Q^-^$;WUW)#QHD9f6HZjIq&FUD -z8N!TS8uEM5nN}!7^wN;u3%xYzfH1k2A|&L}rT5k60byJpb#|`!oD>mtO}s;St8u}< -z3KxLAE6%RwMlK?H8gV5;fG)Ym;`|rV#eM0KH(j{kMdB)_GGT~wL5m25e#((HH&8;f -zD#hh+0Ls3Kn=H~?hZ)9y5t+-_Wxf1wNnrfOGR6nYVFNjDIhrfK1Z2+30WBtwhaD1& -z0AZ1HlVGt>N(lUZY;-IBooLQO6^lZQwUhjamWJSm+VLlqh;^rc^0K@O$GG&`)~R07 -zqJS{PtB0+oZj^b3{$3Dbw2+8IT#)DIUfPwvpdBLdqC7)?`X6W){!Bw#2MOm3A{=9G -z6q_X#!``J%?S -z*w;%N?ZwY?jyFfx-AluxbxF>&aOhrp%dnoF>r5Lh@O`vcHo=(|B)r{68^y;t(yAxYJfW%V<|dihnZDdavoIx>+5v%XhA;Pg0rmXpRRt7v3U`9N`50gHooDUI -zJ=V(;@`yCg@uCXZXlLBK0>P7oj!sfG%~o767aaXm -z+4KZ?Y1$vh6y-rG?7@9+yKI?U*4U#z+t1o~Prs_)reK_K6Q|1ffFnvS6*q#2)X7fu -z<>CscDUt*l?#s=!BhiJ*vyX9}#u4<4`N~veT(Mf}F-s(tiU`(a0(ha$}3HaUr+DTX@SuJm| -zUa-hL?SkYAF5q|CPhDW|Ne?Q9dI|PirQHkHZ24L$O@9gXqOJ1w!Z)@w*mFbyp(}dc -zCR|^*XUh&d_Yx9}yEHv*Gxt+V@vxm{sAxGr`C{AY%L`APuTOsKrnz4Tp~J-$?%*fC -zeWQy0^S66SXm& -zHtZB5vH~nCwe=EPV6k?kHcR>r$qFFIwFY&Yt$0G#T5T2>%L-_dh-CK%(4<=G2utt? -z`;Dz?rwut3j|-WoZIiNFirl}Ebq0;QY*7ESqwY?2VMO;cRj!cgZUm|BvJv}tJchda -z*pt+q+ROa#*i(+WtCun>kut+Q3ouF8mOwBMKH{YB9bb8E)fafyp`xDmGLMTswM~0! -zuNFN%t|Xv#EuP^tAR&WQg-tZRQTIqFt{x<)baeG=4cOq35AtBA*3hVmlfDUS3-^n!OzMye%=*d`(VP(?9kZHO9Y2|#n#OS^RgR6r6RNCrG*$vj=O -z6;C0d9NTpB8{X9qlWI?(9ZY`VT>b~umD_MQDP~IU;-jy -zRc$Z9#lHckYKDM_nJPS(6)`j*Vzw&gwk^UK5HU*yFA-$^h}6pI?EC+i4)(p>#Wv~z -zJnpx9yAmhwM+XgIMjDtwgC`CeL=B8J2JTn9%5|;{Dfzn=uAC~)R-0r0t%d^9GX`&< -zRpv_)>Ion2@A>jF#2)wIR_DuJ8Tezwemz@}{dRu*+xbzA+F34t`GP5Y=IZyK@ -zZ)z$$slt@SpBhZ2OmE5+Z^V;DRy^!Omb_Niwx@Mi<_`PpbSFt-eLh4!vEV_TX!HU7 -zT^%8a4L(s{YZJcFn*Oa_>k?~RQ?=1P{vk_b$P9dNo_VHR-grw$w! -zkuN-!O*OYF>4QE9up(cG1jrsB0T5@Zk_(5hgSRQ_ee|2vT%lB6q^UpQM)~d7%{>e% -z-R#4eLEu04;aI6^TA%>g>x1h!fozDL{NY0(wR7_&BF-y6_TkbHfkq9|NPs_1!g}#z -zNLVlNA%ulPqo3?+hS_gP=yKBRrO|oDA+;rCWESD(>dnjMHnT1N^MiIn8dtL3o8(YzgF`fyV&qOU^5>}Qq-q8>6 -zFfR{;xJTy;Z8>m_@Gl=sa)6Eb()Nh=;RfalViy0257$3mXwPEzAT`hjfq|ve9C51; -z^qPS6`H~w-(dyIGMPR>tp_!z|`854XU~f4+i;wyj0(#n`zvH9+iNJ35=x_U|FOu;X -z|9~`(=u>3FG*a3y`3LvpQRRkcZB{ffT`IksS7KiXPYv$O=Cgut<>4ry@?=)&&Af_e -z1Bd`gA8Bsne$SI-L~$dyhhGfj_*-5$x{de-5D2rw$oNfVjI)@qBql1GNhoAY51N+` -z;(p8%o;0ERxzhhcK<;9mT^@anK;P#HvrUr1wwi$F^CTe~qoIm`XY+*V5|ytc;5T`~ -zR4FCCf`F&&De?aR1a_L`$l|L23C~sB0dkteTL7E=Jh)Ht>fl-b9YBy%vK8*85!|Oz -zp~MOw>iIDmY*jzXqbq#SDp_ed_ynxWWB=ikwMwU^6#Y%ZxZP55=~sQwObES!tI1>k -z?jz*nkkHNCzw+1(K2SMsE&Upk8%$#%$n(97-{H?*3Hu494C0)ONdgIL<}g6=33ld -z8`jDFGf$Y4g+27kKGsCV#?Z~LfmqcSgrz&VzoYsPL@e_e6&<_Mn&`4!7dvj-#9bR* -z#@PM9;Qn#ov!#t&6;)P6+?Lf#ONdsCEKu7GS?j7RQVi}UhAYk!LXk|X8~vOQw<=F~ -zhyW_~Ss!j?o-m34xJu*G-^8@KaZlw53^4pNKF}mD&l83a;Au=}+>$)O06=`or`4VE -zo!0Mh`71Yo^8 -zxygvbKMR1TI!V!tT*x_+Pm75wK9AQ-3C_eh=5tqSv@kZWcYhp9o20Hv#vdrdohTyf -zBdezgZ9^U7$1uiFCQL_^5M&AA9?0W!q#(l41?ZZC@_3U3l4+jnpC@KZ5Eqaq?cc;K -z2}D}@1UDvZ9%D>p60#T*CcFMjlsW-#m%^0g#?W!`o_TR|Eh%nrnZvw1OaGCINqU@N -zpJbknXC6yr`Xw>+Jm%rlplzljohv7tKs@zT8-B}w!*!aNUv1RG>MJ&CICVWFr5a~V -z)Hg9d+bB_e$wu{S!i&C1<)V#ZVaZ87jTwx3x#@z9`e%8YjpAdzw^1L#`#p7}>AW-_ -zg%8`2%M2aLj1Oj_AE-KfRsUW;>*3$^m;2Qn?$VFaw;cWgn|l4J!qqrCEP@St+sC@v -z-3B@cCXKMD9H^k~9eoWK53o16WHS#XUxDIJw(Rl+Er;Rz#B1pf_4;XEOn`FbYX<&h -zzm=~IGLP(!3%>3_u}H>Hyn3fQ+h2j@I-Lzra0LsPfbf>XclAY7`{6tK`Bc~8+xjP| -zvctFZYpJTkH}%J0RsH*`>&9%*Uvy*q3)&%#{>S}l+=7swbq#3?A~x%?-q%*FGt@)w -zUu)=;hg{KymbMeL#m0Nxmn>@Sx$1~^t`Esdzh{dKCAs%*Z9Hr^_z8tcP#H~W6-qYaXS~%_EHdb+DbO- -zth;QW;Ji}aH*Qgy{RVL>%+jKW6(9_~AM?gyGg(F1(B84Iq08(ykSj2gL&H{AajYCH -zPl9OzkuiZKPZ|v*5SL{Z!{qKypmehsD);wn1^eywh}qMd%P4iiyReljcU5G2FNkki -zP<#toPG!aL>f|bSZiJbvSdb{_5kA}#5+LHKKHy;B?9I(G^TT}f<00u%?hx7{A2Yv4 -z8qhRXxz`nc>*lI3{+oZpXTn_NF@MrL1jT;-+bH9xr1j%Qna}FhuRJsH1nCG$NLwW3 -zvo5TWlo(|>U)OS|hYK|er(iiQ8X%&Wg_HQiYGObTR2rd=G7HBD{Q)2FD;$yO50E5C -z8W8)2nuU52AV|hFxW?N|;DwshR;{zrcNGI86w5jIy~c-0_i^!USg2M&Ha2UQ;-XRf -z$5FxQ2pz@VmFhRr!e;A8Pm5oWWn##|O9K5(hn0jF`f^;zc!e?|>!tA}1|cK_|5OHp -zs&kw!#At{JQME}Q(uynvFOT`%W;S|A-)pcFls?vyOSj;=_(uFqcKHXsD5^uhAiEotv?)z -z>9)3Ze7$4_0g{znT})Mu&FY^s7*P9&Z|eV}g_r-f9}MKTRhp$2eiP$PlCh5XChMh; -z`a8H?G#)DxHVZJ30={eEH?Ue5_G3ttYFLhve$21g%Ee?!GQ5ga`#?zIraM44AG6Vm -zi?D;d%Euh$g(Y&Tgy3E6ZOx^T!u2C!`toc^6cSw=ZMb7O_b(O#bw9G6Z2OJd%Hq5& -zfB^scjkACv>HvuRZxD^P0Ob7$`0j7qCIqCl0O)r?bMyx*D+zCR$xDpvD2-^d1J#!vWs#o@gRMqDn%r*L(AEH@S27 -z7QhFV1~P*?cL$%Mr_V5a-o_{G=`#_ZH}T1O^yZ8DSjh_h^Qn7cr<4%s8*Y*1S!e~E -z2gi*g^C3=u4O!!HBrJdTBJE{P_`o-E)n7y)Tj>85Cv-81U>s6H&6M{fWjQ6qA4V8S -zcE%i>MuHEr{th!6)wqc5C-3!?+~cfpI+Hq2|L6_QXS0Hsi4B(s&MEIE$K7>IMbWHq -zE;GWY3ptllW5w2*zUa+GpvGl@R2kC830FAVD-{GH&Azpto5%_$GI2BbTn_uaH#eR% -z`&CjwBwiqL2xXCGU-;06*vN-ioZq+^QdYORwQa@m9R##<-eF&9XH;L2^5cfF!p%&o -zgPigh&aM`}wWlW-S>logoP?ZTO%+Oqe{;OVGlWd?i^RKkfo!2i(9O2-WCBcaEul$0 -zjc{F>M4BX(%!VUev760%?Yun>YR8u)^Bwz@lnhA+I4_g~qHT*a&RxQZr|h^3iej%) -zSE5bCX|8E_$ubwVLYk|9G3A%wclq!#>j%E>CXd=Tv>{8!Lpg}<3bh5^#2_=pN^UD(IMHO8{a&u#;exIxy%*Rqqo_mIEW%52z{?nQ;ybNzE+J7t*;~R&L*j=C}6d -zE1mPD_0dL&Tpvxz&CQh!I4ly9jB!e!dqd;A{avTAe#5E>ot2A|O`{S%HuEDCSd9UVODJCw>i<^66N^O%pEH`B7 -zOO)%x1dp~D(v6nSRY1W-JahEgn1@8Xlsz~%B0*P>)RsiTL?tO4Wmq;Kmr~~;@(e-< -zo2yuUMoaEdN|C`^yf!x?N|zQ5#ji{!5rNOvNmh439L=sxglb4GKe(4=l$^fSzioBvh6?meVJ5rhI;Cza?eLsb)*u+@WW^ -zCJt^0jE>!Z^v&PlmPPNgJtJm-yAk4v%W -z=e$4P-SDrRvJ>5uch%u;xbE<&_-%E`_i){z>)fTop2yM>Y2hNthec(nslfaIqo-q< -z*qY;bIB`$)HG6(R{=>#;LSN4bF;*CfXKJa(qskUr6+KA -z<92K3xrU;MxDBloTWt+%a)h-xlxACku5F9d>rxeM>!ptrMcp}V<9Tghjd}L>@He_O -zByOK}o(2zyiY8U(X6Fd05;^^n90)IeBVsC9O5`Y{d}OX? -zj}wnwO0n6d{YKXipCiPBy9*8Jy`^h+l&F?Y0yWcH27n-hKL*?5(%8HO{aXe -zB@C7;sKqG>FwmZ&!0&y@eI?BoxX=BuZQF-%zIo({>m@J>1f9Wu`2<;IeDc08ad -ziZ)?Hj$@J>mV>tv#T-m~5P(_yL0D?gfs2g4ks0zJGaYw%2QoQ&W`#eqT1#zQn@%-c?e2=YKn3rPy%xM+x*b%>XdF`FDk>>6`od?ZYrfvb -zJ(Dg-n~0g}bKvDD-2!b9r`F -z`pq2p4TC!>H}MwU2}P#6q-v(?q^A1OBy^aZdZEdwlAU^?qn)HR<$}pkUC@b7>&**{ -zeCI@=qny?VDc;vkRB^I%ypv88e%c@79d)9z6P@GLJ5lH!~?jWg$<~T(WgTN!{c(fOpBS{C&a3&A37gU(%gyjdyWu=5fCoDZc -zDlQod%+?_!?};SuXp(oJdADA8%qcKGO3Ih@K(4~0P6V3Nwh-=w5lPJolbkV9#gZ|P -z+QyzT#Duj;7Ds)hmgB~lT3uy?kBk`Bi}3bFFAwB`O?IhKq>#Ln7WY7JI()D<7i6;U -zyhMXH*WUrq0Q?*P`=A^*+s_p5XIkcG;{5;5`lB1-{h>eNbpcWumE0d)?U39bm3Bz( -zk1lGpZ~gINw$q%UaP+b7oUpWwk|tK~$n>x&U=&QZup@ff}DdIJI#)ucBnmFDwxHj7`#K|xZ_%iNp$!_f$*YQ}Y -z(`*fYC-H@z)DtJO;%9Cq4WfJ{O(k3wC?;x#|q6pZflsBjYJbNE(zh}BP!iP?WGV>=I>P6 -zSJ~;d3*)jQ8g-XD+I}SroqjSqVZP$KX1U6vP`&bAqI7n4{CBAnJ*4IOW)57EvNLRg -zK}tRSlHoNSH6*mr4dmcN6F#ZN+$Y5smAd*KvrrM9>;mu5U&PG4T9akyz`|#@bL09@~ -z+YC+i`HPSRCfatY_(`_#d#At8Lb)3=F9{dPM?!i8a#Zx@$c~cbyN7HI-)0GSvM3GI -z)V;E2i=!$~=YJ3yq_+M)ervagSRuo7QDex2FdA&fG=wC(E7Ec6Q~H}2eP!z-)k8Yi;a7xZ?Ggbb-F -z?S1kP<3ETQ@?RTEU*L9rD@OHDm}uG>sz}y#8JD;>$VXHfyqyWVdpV#7*c$$kCG3={ -z>cu}-Rn(uV%090BgURAH6DLRh51Dt5z`FTPmSgS|I#Sqe7(y^fR0*5=&_AY#JkUVC -z-iN%9!oMpase3|YANpJhy_Jl@Uislq`K{>-d@6;tIPxp&L!U?y-Bjc!!vGkD|7K>w@0kI7eqUksJxy?2H!QaGHV?p%5cGOT~py3f#$>TEmvnhgtbRQ!5uhvznvUaKfm8DXms&|Y}3SFEM -z*|UJU5r2IFAtwDWOsz!@cY6SZC6T>Z*jxiPiq?>AYAt)mj+XRgb#o1lwuVkhpG9<_ -z^y*$d3CQWP*fmb;=}A>k0kr1+0dTC?ZIjt053-2Ydazr$-V=12`-<>Um3FE0>;j43 -zgp|?ZZL-}ZeVD%^N5Xys)9Y14V!L{Rr=`T7T#$W)L^LK7mW@fHrM)fR*(0R^tjuxJ -zXxjdU3GN{MvcS-2sokopZ`Pftp(sfij0--+)TrKjO33788+nr_6Sf^l9nhdcS_GJSe~RmAAW19S1t!RAWLk<#nNFmY~k=pA`wpDH~rmwxu%k71wyqeVegg@3hKZlC(RhT7W$RCN~ -zYzpU*)c!pZdBmT{GvzjWQ*JXPA1noN>QiEjQ=h_9OOzM=73v*p-jg;ecB(95^K;}T -z_3tepbyu89k(#4p?fg@MmlKtZbZSq?+oOf)^BvuRilry_&PU?MwoGY${0MUgzmrTj -zyU2vIXFf&*nQ#uw-$|UjMWuv@b@sj^X -z&LPypf1-|&sVfNe(?3$_(Gt~3-b9}vAwT?MNOpvrjzrz{M=C!|P6xe)#NPHNI{k-y -zgm>ou)tpQZk-|ClWMU;6Ppl%1H`CF0kCTt6l!q_~P-nucf0+(k()FDq3Jdz$^<}B; -z__aM(ADIu4E~uwhqI+TfpCme@k8RE~JttM9Tb_P;eoznPE3WJzNrNVh@{*ngWpD&tUTAchB^SRW&U+8HVD8+g8zBu#emzMuk9CYG9^k}S29~szzQXdqb?GZdk -z=i{D9cAov8n19f%Z}yK$QLuV>EWt0PvOc?eLq{aDN_(=BOp9#^)T9&0w#1P(>d=10OcsHv9g#WJt*Qvh#c7)E6L);IYY(FJ{I+>zqIcpLu`WzI^#KIWC({;u?}8 -zmuAKoJJ?WlA>5!KT$B2!=yLq0p%Acad&wB02i*vS5Q|0>ND -zQyh8n(=Y>M3Rxu2WB*5a@>6;ms7xX8|0YlLzp14EP3xih1=}(rZ)SkIgtMqNrK~%JSs{fsNn|2M9-W}Nt&INwW=OxIvQla98fT@Fmp+|Z);zcPS~8qN4X%Rs&D6lq#TnV=3^+`L -zFGKTeWz#T4JKiOI;DIi@RQjc%46i)R46cWB>Epr7W@U^)anOzETy}EoK3h|eq6KL@ -zuFp=-5S~wECKocZvq|hPG$i)-J|y-e-({*KlE6Y=^XBP}cA+&r5g&!1$lj!KDOJHF -zdE1p;k%668LB4EXvOF2{Eu75~<(R7G4@_#@q6 -zBYv%=Z(6Xyds=Vuce>kJ>CbXb-(F(zZg+2qrR3F(wumFBuYf0*_DJHbZUne3L*|FEgw?uFPYTlwZb=$|(A8+LnI^vW0`c~0R?N3Y!T -zSG`iBpRF#welMr$aF71@80+C~eN7+}J*3hrqY=i;)B`OrW-c;to*C`1XMWpQd#zq? -z8(Ic?=3~QxUrK`Gy`dwS@iUm{K}_zVDujK{(~5wbsoKN!`VGD<2zbRWcohN$KZhSu -zhYcPXrw3yU@#nhjTgR7r;_UiuC?&q`Mi52D>43S|wECcve%p<4OZgSwDp?nzFmhf- -zxJ*S#uKellBF&E3xMPNSLON6aJ^jM9MCyC7g~Y%x-^{^B)Qs2L$d=fR*svq{Fa8+m -z$Mg&L5-A)1r9Dy&XZd3!`tv_VI+}jrH$-Ba?UD9zFuS@xC-EqWw9kpX=JX4}bE)6N -zW8KtXQY0!c+H!KWqM55Falduql0N*$xbLT5@Izes2rMK>Tpz^!ZMEW8uI>J~$j-HN -zSGvJysh7+*tU6-uB$~+FXKb|3edEa77dIrtV1NP)w^dYzFfvtcCR3%bHIX_mF7Q(H -zljcfD6&&8DHq#f0YvDhJ-IYk)qci`vur-c!@3CqBTUeQgGl^6OKhF!3H8l{7P#%Fv -zmRDB8Bw1qR{Qa!R$)7+JcJBWe1*^&1wyBz?dg+`W``rz~D`~y+{Qe|5mMn{8P6wr| -zkMdEH_=H_}CA2|Xsu}OUMZ -z?AUxGY8ds56FUN7r>?ut$z~%uY@Bg~t2X8T1$INcHwV*Uf>n8&TuV -z9}>LM$xiiMyY_I7tUdH%CrOoTAIq-kK|a)z^y}RY^N|0G^nXutPG8neQqQwt&guV> -zM*WTQr7P}BpZh7z?C*E~!Kyd{y@Ht@!z50p+{oiBIkI;-^0`7qJ$@?5u!E(3p@PVn -zv}`XWj`AXi;#+aw8A@IbDOsn5oA0x$X&Na}{p_~w=tuwCxkim!e7{7@;9e2?RvTh< -z=XaBpriSDH*?l@yYO%~q4VmO4Dq{cyyH&jnM{nr9kaS;*)urJO$m!sXbUaxn@NtwX -z*l#Y1=>G_l_ym)a!>o9=3|B?AdsH35-4##kp&RR^ -zu)YufO#y -z=5r5b2-D#&EKba7gqdp14xDGJNo!NWg0w6vP_U)UN}%0V6Vz**VXJ2U4Rf=<`Uc3tS(*62cm++m_qt!R++A5yJ&;z1h0IL23!TB -zOR)zqzUy#di89R*sXl^pf5X%` -z_|-&Y8cG-^x_u!uf$!0faR|PJhZ9NZ_-+Y74l2NW%})NF1WT+-jPrLTgwT>mZIBq} -z@5nL0y2LntTZX~9#5jLTf_oX~Z_03^4i$zhH?TJ(gj|+~IgL|$c>K5a&^Wcn6;e?Y -zj{F_%aW9_k0Peo_xJ&{$wa3pSfOC6X0z!BTg>tmVPb7v@d;CaXr}hv5JGaLVfF12| -zK9xy)oXJUNRupjGrRsCs$-o;f4ZNSxuWsny^n{gw{{YE~O(nDt)}8A)O^l||pQJ{iTQzFjHC72lj#d%RX<#J^%x4Kyv)rM+ -z)8LvOVr|7TD7VbFn(|A)kfxz(-lwMUXUT&LQr^_($a^$@Mg#8*@_a)2*}qV(*-Xwn -zW(CW=o2qw}%C*;>OehPba(PMR>QBlgywzK(H{$8Djz*UYMW2!2-fl+!TY{ZR^o@iN -z9&r?i{#r(?ca(?zN{-T79PzY-=q-(SN`@PCa$&?TC784k_WRSixu@fW?i9?j_QODz -zRD9bHln_SJKR(>ua65(QEDPeDb(VRle|-iOG<;39Wb4p;2va&0sCb{FZ`+z&k)hy` -zOa;j*9~y6C$o3I9BTP!I0k78mzJ4a!gT3%Uz4UW$7(RotiHrlm^8_n>~F -z;lcsnN7?ehf({G?m@2QLNLg -zD|Xse!STS@#Q -z)Zslrp`XMkE8ykMpu61QRCL0B5p%m*jdbgm8EO}l8H%@yKo~8x8y$12-o6@P>!8$^aoNg -z%H?9Tf>|#bS5~DXBU0Cq&yc=hocpdJBorTEbN?le?^1LK7&^Ty9~*|HD6(&)ghYhi -zZ_SSQ)B(I_JZQ^m#$V42@!tahuG;&kv)(|nT2 -z;r}U7aF73>>T$Q%zrkZbpXm4pyI;`c)hNp+D*j!rWX1&(ySE@>*U+EXHTW^ReS-q` -z_@l!gbmfoQB*{`usb(k7+omiZsn{_7m`)$ANghYW%P|}GT8e$VyrSVMQ-l?9(s)@* -z#*4-=UN)qNYc$;Y6vuce*Kn@@AmhcI2Cyat<7K0Udl`Q*KGtde1u;sAv2flNXw0j+ -zK1Mz7U*0oy+g0p2TWoZoao9h9ALF(XX9F7cJA_zSg_nrM&un3d6>nf?-3*VzGVLsAv$5K$m(c7I&-VmIG& -zX{)cLHN;j?Xd8l6Kx=c;UJ59c#I}lvCTXjvp+dU2RQe!o1Z<@k+p4(acjgArzVGLK -zzn|Y9;>|2)&YU@O=FFKhXQahX1IFBWJZzfFhNcbu2H+3DOwc57cX;LTD=V(ehaQYt -z1pfVpgls$NL;S4wuFi)(zyoh^_Iw6Pt*?OBjNz>K(_wj581Fmg{b1QW@kOr#@dfjt -zFdm)KR^6B5euW!rd^DGjhwsad3WxCi8Txu^C;WR_)zH_ufJF2=cW%w!JylFZA{ZEb~aL^^~IVFjCi#Y+~0# -zfgyjK|9zW|XoTm*c=ji*)$v#p(^wyb^NrI$QHJJTx?|H1HHVKk?hD#{V`$xkymPl< -zkk8*{&SjXXWkU-N+y$V~x9)MJPp*J{^ZK&?uy2|j84&Vig%mdVe=%_WG_+QrHHvM$ -zjrNwL<2JerkiN{^vkdM69L@N6_@=}E;x2&ghh-g~WZ;3M^){M3+&Y9{yFbEVN%OH~ -z`7=4y)$m}sDm5|ef+#9HA}V@()DvM*&xb_48U((N;j*Y|f6!{exob$~%$&pZCMi6> -zFwEB7OLM?ReBr`n5VbzTu%{O54NzG>f;Xh#X)B<`$pDha86sgK3(35}@JGu! -zrsvdu2pHLo-V@c6F8t&Q#7^uPdv#h)@${U*Yw$YjwoNI26Tv%-TRwt|KS%c3h@1-t -zz_yH#Za&rQ^Bb{+Ybqgbq!n%qtH7a@dzTLose(DdA}<!@%VYBAOQqb70PF+XM)d^KJ{%({4>sy)^o;s2jKer5Tf~OpG#uERnJo0a_5_7q+ -z-UdH*&b3w2n(*Ku`G4qJ3bp1blrc+MIMMYuyaGi%WwnK`kU%YxcO?yMoZs~h|Q -zp-7vEUg&kKbL4PSA9I(z*T&iD@CoG4x68KsPJkgY&rcBa4y~h9ZVpgLl_`XqbWGAR -zh5pmEQSZQ&D7dmY0N3c&D~itRxma#Rz1sCbtGQc_S%vTTWI~u`A*!7~z6}>Xi$gOY -zbWMJe0*@LCza|a;N}dA0f`)%>o^@YIRF{4mU=?GUy6kTX71yJlIIT^Hur1Z<1>{!e6weWH$jIR -zwT#m}CXeW8oDL_uU(_i_@?gYwB6(1G8=uWQod+P1qeTJwNO&yDM>9_MEam1k`_66K -zV|nJAd2oa_e`HRNT+RWvrT;iW8`2?vHDAda7Q*Hm8Se1mc@4NtH}{O>fj`Lj5!@he -zp1JE@UMW3xB#&5?qrQN!jGL^LDYq;a=Z*Fvk;rLGK;JdrI3PhF^Dmq2{Fby -zwD6kiy}05Xs2vI%tLlQ4YGFyRfx!UZ@)}fyG3NQAg6W>?aEnLi5UUDRpor7K5S&`G -z8x>5T+lSlQo=0uZOEHMn8qbP(*h)hV4#F5N2!ht&h8QVe@^RntlD~LwUmi}TfA!)5 -zO;`>@&w*S}C4TXeecp?g*?w<|LA?HXc-v&cz)FM6@A*IE@yjaC7C9#j6uID^XQ)lk -zY2Z5GV$fdy^hQX@YhKdrExW*t%wl}5`*&___Q7@>qm{936BCC9SXV?$i7X&GIxNdnB -zI}KynmX;@?B9_SNqB00|0qF@|s3_zi00-Y@EgItZ`;v2c}p?j_v{(2(cG5UgCEC*5RVjqSy*Ye_ytEh;RD`JNt -zzH98Lh=qL=axBY>i~If?l -zxI)^|T-fP=L+&X_`H>6&3@;u`mo*NBr_!4C{~M$fN`8(LOg{dK%|v&|-N#2@tN)Fv -zuHF9u_hvz{J}=lf@@B#QjrYlxd-xf0r_1oCSo=K~yAg~RH^faYZs2RY1tnfV^3mKZDzi+kRv0Iv04rQDBGZa$ -z4jt5>k2cR_a5N@A8>TTwUgo)6y6N8BQ&Xsoq8(Y2cPV0p`ME&@%h7w!gIDC?a%PNY#|L+w5hc!hU?69mU6$%vxd&NE2Nob-ZtgI+A;3e8O(GT59c*zIh -zzUCiXy@yBYb@}RXt^F@RAQ&@a5DaDH<Ioov920l_@D!r4kMcn4P}1ka%LqT(WkE -zBtx_#aw8zz3e8BENg%=410%cv<1fxdJjZ@8ltw|EKF&8(6AYV1?hbS~JsREvgG;j^ -z7~@SJ3bwpk!rV^hmaG8xE(vop9W{*yknru7YiO46&A*xjQrl`%i_ -zqfKlOhCjguV~i&UaDudwG236vz&c6YLf)(FfzW -zFOA-OUz$CKGfRDMI;JCvhjizJLG$DD+6 -z;@H~!eh^TXwswDz+u-JYdJe$!sUkyTo9g37Y6KqCP0HcqLH#AVM6 -zB_iIOoMV1+DUkW;dudMtlrRxQk|?^4hp!TQrfR%k26`B||IS!)u6}g}c6P&{ZOwE!Y2ZwO%GPAlrwktO%PXs$eMv`ORnhw!W -zGbNj^dl@mzDCixTDG~qfW%z*IYJ3TqA=y0OJKN_oceA+z4EIt_F$R3$wd=uba9|j( -zO09q=-WBLwcye_OJTkcv8Mjs6SHyPeof9zkVoA{;7N!EvgFeOx7$~{WBBzUfQ~bRG -zIg$8a_53HSDF_yWagezF;bmB7qXnKnp0TD8@Qht=bdJSXmBcxgXFi!-%j3?1-dVyf7-{4rV|2t?eJGoYt|*@7Q2wuN -zH2s%0=s8>)mf<#DUP90Qf3^WM7Ev|XYkp0KO^aNWR)L9pWRC5Ws=njM`#!dE#`LpO -zrkM*f7&t`3jjYiP=2mIZZRF-5$9`a}ow4nJK15`(LL_DkD{3n0SL~_aB9?NxNYrRN -zN-2T3lu8 -z;UTHbA9xvT8_x=!C@k24eH=A?Wz00@)$Dly2VQ`WmM|NDe?W?Mirv#B%q!Wj^e&M2 -zsv@LJ@}!vD_3s@!4oB87Z-(a%;F6z>WH=o)GCzy_x9BD?E3?h-EWsfNvwO0HS)Oe! -zg&(lwPUcV^*`q=I$Fe!jrj^5UluP6KM~uK^dEgcXLc<>qFix#tG}($DHqnzH8~TpK -z0~40PM&2h&n8n#<>td)9;Es?mi?Yov+z%3dEX?*9xoJ6q4)vJ@+2+q68}`SY0{3_c -z+_#^bZ8k#+Ii8d6aMU&zdN^*l{KMJiI>==sBoHC>s4pS`zyZ8tfoza5W(04r8!R!` -zYlhcTcw!@DB#&}v?hE!)ba0Ujd_YPf{R6Jx`tl%0e_sxL)BoOjhw#MBwfDf@B0XG1=mTUh_hq>MxUW5sxlHv)-$;?f -zEMCm)%!1Pn)K~JF|9q)=JN!J1eww|h02PZ2hVaTmeK8NF;&?1<-^)gTu$TQ~+y5~C -zp30OA!=d|zaRI*Gl*O&BWbbg=Ol=m>Or4rGHN>aYPlGE9fH#!=R~FK=Y4->XAnoNW -zi}5ZVLH;~kAJW$NV7guRrM>7Y$9CVB_G}g~1-_f^x-Tt%2%mosr@`hJd~d?tx4k>! -z;IN6TgcA+=hIhvophUw#GaQ88^ol`-NXZ*e49>O6d#!5Y0I~ -z3&|b!yg$^s23BikF_(b9&7Sjz*GGM0$|Pr@Sq{SNtUr?k)nPfEH-rA(zdRue_uo1) -z510j6=AV!hx!_MrAb|D=PsqWFv48&-6Ij!0JaG5^pmYrLP?q_jETc(xxSm}TK*VP$ -zqV?cG`px2@G2_U`{4Iqxo9sN8i8`{iHC -zc{JuE1g~YJoC=le*O2zVl;ik`O+|Bo;%}&LSFis77_+A?Rr0Yu)*np|aF8w{?$x>&_}>uLz&2v7EDQ8Yd+9005j*6kmq%|(X+u@51s6}O@a -zOa}0gwW7-R6OC&(PklfGKOwRkSTDHrA=^ZIL2%dF;JocZ^*#-;PqR1~4vwPNgX@9v -zi$`X~jnVNj17E1L-_ZI@(#Vm)(x?35vA8c(CO&5I<#8cp>-^HzL*9e|^)3yu%Li%m -zuP6>bK1Lml`WiQRXOYftxkwQobexA3Rf0KZ*sOsp06y8tXWrJJIY*ng;Xpiz-#Ta* -z4NAnO0A>rgSU|9Kqd(F&knV%>|4az+IXx6>;Kg>6(Y?o)J#6=7ieR719^evDd1iwK -zqLN!ieiyZF(T!qW(x62h)Dt+HL%^Xnl>Iz`I{*RpT%|#a2wCN;#^EJKnoaW4{IEEo -zWxsU|t+}Wy@+6C~q8fDE8qRrN$L^yc59$6Mt< -zS@*$RGna!37mj_%%CTMGDjvJ<kF&zNlo&rcgM^xs?eW>=YcXpI7RB)i?gYT+lWe;qgUVcS0%L>=`bF -zzs7DDDRm!hQ-B5u!Zm%Fmi^fO$O|t+#t04W;{|S;>|=(t_s|^adnUkj3h`fyo>}uv -z=EZ~}&t}98_^*h~CSdP#Bllg=GsBSQm=JNL7|H19#V&)wH;R5xn0IBH~y -ze3Hp+Ukit($S~|tjCTo|((wKY?W^vaTg-=<<{wkYSBK>;4hxKA$Ez6e{!&G>bHV_( -zLP8sCiGOibCQ+3Mn!Zm?Fksq(Cs_vZI-)6<>0KHj)0Oje8nOEZ?zD}oqnb3Xa%ri# -zR_4+Kwgw!MpPNAU;B`z>xxIX}#WvNh39FSIl9PYHWl*{muQ?2dKCTXuB@cz>w=iMf~bX-$!IK%eSTNRR>>mRV8CFgUFqJw7TSJZh~x -z$`BEylQ85GmUo%k8D+ulXf*s&!oN89m*A^MVUL(_>1^I37Iw?s+R^kaZ{_Hv(x_>n -zQK@0NlQ+X|1ei|#8MYW=Z!K}(?!#s1a)jKm9Q@Yj-W2n|rb#E?+`v<;H*Q!4L6ZE~ -z!@nr%jkVG6c-<=LV+D{TVw;AN{(yVBjxQR-Eq)M%K}0XPxfyyk^o#ua%|pC4abeoS -zkF8w<5tzW8;PYQzm%`060sGl6tE7sL0VtfCr3ODfvyM=FZmf-Q4*;nj6P|0TBgdZF30uknJc6jla4;m{L}1iscMbKM`w@LNlzu@5x|0iy3ZbralE$b!Es6w6=!~j%>M5d+Mzz4j1Qo -z5|_i}BUcZ6%%7GpC=6n&AoK?I7Nadxz|Vr58=NzN#XevN9mM1+jAh^rJjgNl97F&f -zGTbvCWVol?*C)xZFPp(m<*eVvG=9Dm|@-pX>{0qY0qYucR(6B=DxHQ8Rl|GbB7G4aeePx><+uH -zpU}pW8Qe-j2M?Ds$0PhS8Fc79{LIs#3(S%XGUQ$wXERC3a0lN9Pt`BbaHNZ)-%{v* -z2bvgY&{YxLbhJ*Dk5UTgO)3>iNuZG@!mtiEC&Nr5eEmo8px`I-aE4oUuT2wQsN;)p -zi6H?qB=8Ig0#Is=;8P%66k`HCSeF1=u3DEMqQ}5(H=wIKy=$q|UEUQIEiB9lXCgBY -zQ;;Hmbfo#TlnG0rg|O8XOR>VFa3&%Hoea?ee{`S-xua58KIDSr14ED&kDsB1Bua*AFzV!pR}@s=mTGsm$tRb4oIPgud&C;lyopI~gL -zh@D@N`E-R99&4u1Xwa4yfj#7IXb!3 -zUGt^xH@m`K6%{6q!bMJiC~PkjwtpuCy$lx@X3G-zm?}@GS|~&a!!z_yMT}q(A`RW|SD1+70tA=eepa0J~qk+%wmg8u&HoCtL&WA -z+MVCtk>4{zbW+;O?}IH#T60?c_b*~M_?h0=oBTB8%IdS38sgFzBUhzCa7ajBzmEJe -z3lpzeh#)%=>?B6Jh>#n_wdHj@lW}(?Qx#ld-b&D4_+lwj7#0nsnjT-_9Owhn1h_ -z{&Q4|^inwg_=DJ`Fn)482ADL64$Msxe9(-%GFP{z1GPx1D5hcQ+a~r^Z+{( -zH)#^shx_awDo%vYaWcQRU$BPVk+>Lr*JW$iM5%{S4Gdbt{^qY^rfauZ!z%p@#=)K8 -z>nz6BgW;=)ndvxeHPVmsFd{Y`p7xGS2e|YcScus=*VBNHFya*bVqc3$6`yUlZQSM1 -z#z%sC*k-^W$#VJ@JsDprH5y-H^{xiK#STtt_G&Dqe%3yrTzRnCGw)OjA2bpcMi`mG -ziSYD@GV}QK8Kr9 -zU#U#Qdl+A3G@2olcUU-b5T4t{BgbEkBMh~?;=K#YUtMS{vFgFyuf(a1Pu%g7zlGRt -zCn^paq>dsJQEnmhtO=iY0s10fD}V3g+5WN6`jf6TpsRPCyN}UV|zF*NMT~H=*#d_xp$PoklGl=vG|T5!($!#TG-5 -z<2)l-$Y%C&O>cKs<0fK&9_sdEmSa2rLi&JLWQihn*Dx7?)^UeX}`-2 -zJllf{%?~c@mEH(1t?&A`^@3xt|8n!k2d*^mTtu*r2=Nf3okUQhH@mSj(OQyi?N7A! -ze%Sc$58HpRmUgy(OLQHvmNqwjO0=K0UT7`}uaq{HSWi{y-%HfKm8g9)TfaTKO1c$b -ze|(_(9@Xfe%d$eb5)cEF@^<`*=CiOtjK{R`lvJ_%PT!@U(__EYueIIAN699;I{2=y -zZ!0NnkkVKiSQ8ony##$*pxccZVm4Y%-`4BR`Vw7?V_W^%-F=*6)l}dgF=7_8b9R=OcRbVmk+T#3e{bwsYcmQDE)n%6V -zU*9{XW;8rIpu)B=TOL>>lX) -zG&lmsWU3ZkCYosvy5E<CflX(~xuEKJe3m2T!?C;yI;2S)|taf8Q9_XTG -zqpS}Y{f$HqHA~#+wGa=(Q0(`OlUI@)pSw0$=(rujsTa1{=atnK;vtFjA!aEagc -zCYO01F6PbW;liQ-knv8T!6`6M<8+cD&=5@2c!T71@V_+K@=X*0?%FzPB32ctZMJGV -ztxhW3LdgfJf(_Iptu|jrDIiK@&f4z;@Fu0-8{~7nc+4tk)^`Gv=neYWUU`z8APjzQ -z=tOULh>6BaRwpsZLMWO9U7gl}yRbIIraT*v@|?&($Twq{fe5b>oR2&oe>3D}V%!(w -zPE?p)M>sZkOj_y%c&OsB9fOzC-jcs(pdxhC1Xtjh)>(E+IX_VD0A{aw5ihBkRqH2< -zw>t)p-}0WAA*=lS1;@@0h4OguZEx&RtA&b;dMm_4kx>gFj5xTpL%@D|$EeNz7^K;W -z2s<^|mux2{ySKxDOm-0y~7M=-0xiW=6Q7$;9M -z5hNh_LjB<7s3iXJb1)j?@g+K996rkAgbDJ7XHk)7(d+aEA-A{Y@e-)uEQmW!g(#`8 -z0wQdNpg~M-8U?S7%D`#Y-$0DU?}BQVK!nJzo$r`0mM6&_0}pJHSGj^!`fZ!=QP#to -znz~hko3>u{P(dDoz=fDMco6t3G~$6QA{2ZsP^gF8pqdo^@m*NhS5fPo{Np>}l|u;F -zxmn`*MyNYubomPm{(NnJu)bem@aMq?Ttk3w_|Wz%bY~i!z5<=W`cAGuCs&|Tq3sOT -zc0vKG-B-X7taoq)99#j1LhA_DI-o$i#aEy?Sl_IGmk*&8)CBjfZB~R0hRC*sjSDfH -zx)lZolMwCZZQKRXZXQ4vIl`AmRp-NviSQ?*q63*XQ|R-(l^yVAm0X;Jm8YiQlkI1^ -zEU+N}Y1JIu&Vh8|JUfmJeV-Wsr&)V|z!ks;bl|E9{B;HJz_jf9hR2#V&&0YvOIiJ` -zv8&VQoUuM1{&ouOZC%c`0f6k{Aa^w@TsRM!0LbtSAOBCV<;oN>IKb^mS-st8tTGy= -zZOMnf=Y_hHM#ITQkH5eKRQCphCocRoaU>b~mO#2fClOfB=zRM}_-m5z;QkHlCE*)3 -zUX6!z@Ir87i_ZGB?)cB%mVh?&LU2{bjjJvqsHEG3_5&f>y%k;OD=eT%P-6y;R5aNx -zE^QNPFGOm8iA1(Ccl#(jJmP`lAsD+~R;O;#)Y{1XGO#u_t*N6dwxC*>&(KH;2kMHO -z%1y)=h&{Pq)+Th>6b9Q=Fg}_Qm>ZJH@m0Y$dFKLqCz56!0x4gg7N1-IuWhDu|dTl;7PKB*ww$8-Yt&UH`{lu_sl;YR*>l5o{qJ}{I -z{6oTh_<=`wTyp8!`BwtFuNTlzyZ?lR_=Q55 -zGLQcUx2MNNhHq8##_2yG3@69wOA^Z$q-k@r+usVVUQ}9V%xyFYv9il80_;?zQ^;$8 -z5c-ksWusz}mTh+$V{ZIl1_It@W`1$*ngTQr9~{sq7-B- -zim(HlZ)AZ9Y!HFh#=PsW5Po)f$@9hxj#gU=HK~UHte%(Er -zD-B!Pe7jSScBItdl)Z6YDu-<=JUx_P%U^5D#3gtlEJLtPPXga89Wl{B5uBQnh{x0l -z;f+{0d5xL(JT2}uW?6T96?|z7fAbQI$x6Wa=06wHego*{z^9uxVEjxMWHQ!D -zl^^yhVw{S49g7V_KT$I#%*Y03)?4X?g7T|s*u1Y^J+pczvPwvBzXW39%sw^lzsg@< -zk`22SXho$r`#mRy+c{!-+)wCXD%jyfn24ZQ0C?-SLsw1IgxI}0_uy}>-7}=I`*if+ -zZ|6EzNn;yy?9JbLZA+!GpXkV&zkz6v@DLO5rOWp7@Tc)dmzj810`M8jYIvQ2%Rb_@ -z1^~2L*|1oeKrO6**j46wEF`3Pv-8vgYw+?Df&{N&bUGgBd`t_cB8zW -zz7|+OOkGRN!1*nHfF}f=Lg;eUW%Q4}dc$O}*c5hKke!OLAOdiGZ^h})01Z3zJEj$9 -zk4g{~Z~%b_aL$iy=Wv8i5&NAaEjMjFFv2haoJ5QZ2*Bxi)14dv48#{0CLT+ -z?~*I^?PkAKqF$NY@%%s1nhO)aj2~wlOfa^Zj7AR;UGP1x;JDW4wdOxdSPPwDzXj@u -zXjO?d?6E)t5gl6vVdDZ!M6`v`9}Euwd&JprKP5H!eZE+L89r&5x5q`11}X|4ANDUv -zO;{vW6HJd4m}jf)1Zg0mjQe!9H`K;@P+K`i)&~tLsN?5AgDm8w4H$ -zA1!!EFdp7h%WIHT@_OocGo)=|=XY9@@mLF=tp-+B5|hE$Hrhl?Gv7@GWQ?r!;}|eF -zxh`nI0B88*lz4hD`eT>^SMSUQ3ssHOSGR1+11zsSsA$TonfuS)yjMZpD@+=|1N*vfbeDNyC7BEH -zt$32+$q8FKVB>4}u(zs%lt^Ef!eVUsP;oFk1V06Ds4scZdP>*ZJ#aC&tF7W>M{9@b -zn_yf9R)Y2mgp(R=b?8rCg|)k^rS>YF+>2EF5P(+O!7l985uoD*Z`m(}ISVKi1s(^R -z9r}c4%K`H^_)G@XNQl(Feay#xPw9{mA#|$X!nbfx@9Y$$by@=62#fQp8U0vP^XOjD -z_wVq!2J)xu_meflEilW{H;QgsPsPH!naN^oUSQ8Iyt3Ovgy7@!H!3_-$Z#QdT@M~H -zO_Ua&mJ8N|BwuTJmu}>({QdYSY~Jk?asaIu$gi0==K3do^8Wx04~NeS98T=nW$x{^ -zLlYB#7|k -zwjxCw>J@*7YU#!)bsCI)tQ*$cuOv#R*lyGscs#I-##VtQW<;+NAJSW^*5OO^C3<7& -zCc{UViNTC}m+MQG!v?D016{!$W^aawjFMy#qnuCPC-bNK)Abk}ffnLHT -zuvQKdB#`p}5;@)n=>>8rg7g}hiy*xQxltS>_@cL>imwe-gakH)OCXPMkWh(P0ErFt -zL3)i{bPLRPxp|^1Xe^5Zl$4#hyohsITiYR6dm@uuEjCe=Fg_puofOLI)-+Kj=D`S= -zp&U18f*?2(jyExNW~g8)4>f7VF{6{oRbq3MdOkd6$c##+Ulg0KsBnjgDvP7a8EG>4 -zg4i61a(`v~lIiEg=H3N(*-~j7)y%XlqMsAbmnKl3vETNv4YE1)?wiuOj;3ytaRq%- -z>a@+XP(gK9Ew%@Tm1{cvHxm_Np+?)wCtGYw?3$_Q93KKLaj9WXJ>e{$?$k`B|A2(C -z7HXW+wxl({VteY`1g_lT-z=~bFMkl(s;yFsZS_*A-S(2n_9A)4i_1(Wf)Xgb)3yP} -zPX?TvU^gk)cLfo1?Z%RVv;2bNu?=GO9YN78_?>W8knp9oLCC%>aJNX{H43Z5LzJ1Q -zy*fgUj}Chl#{sQ{8r^6 -zY;hH2=~6KrQLk-|v^)kDfv@FN2I}j=rc<(AhNrOI4Hg1y2#wmUkzk9koCZC{Y9Wr@ -z>bwA&;jtKpyx&RG$ACrm?HJg$*GHxEH^V#k6X0kc6y89jksfnVR!|spM1;IL%sg)R+=(~$Jg!bz*Eq5fP5HeVWC=pC>&Mm -zhlxV=^|m3RxQKdyh=_vgh!)cx%=8%6A`N>AmRT54AOa>)S&{K`nt}KXI=l0c|C1)q -z{5P6RJsQ)mBY-gSPQzosFD<)qJ7Lu}MK&I`0wtOcd+n6fKpc$$3bw=m3mpZrBm-Hf -zV@|-FXfhEN7(iLQyt&j-3cLcnX|hmlCd!gmf7MCsmr`}LnA=Y{tLpC@kew2nya -zpOH@DV;51Y?F0^pq(#@{aK;D+M}-O7&|xeceaA#?0MIxZ -zg@HI#Gd?Ab_^Rj*M7yO<<5Eut4-YK9xu7Rk);em(4D925f(E=ZFwHCPFHG;lK?I!g -z5GU=#>GOid_l;X65NX0;r%n&X_w4?{L^$lk7f#|Uxg!lI{FR;h;zac>z=56>NWL3r -zqTZz64YUv{6ZKZ($AsJ3Zzllr&DM_z(h74s!9;Co|CpeU^}C2f6IIgnF~J_~w-9gI -ziMO1@W*4!gj^`mtre9{Sc?=!!c`A$QxHO3`{wJoiZGdm+C+x$(9Vy5}9d}YEEYxW` -z^<|d>Z0IL_M7vl56tWOsnh3kK*=6tG+2J~Kv+6Ae`Z@D0D#(1 -z7f1pSYr=CE2^62|K~^4Q7d8kj#2Ncpf%7ci*%oVeSPy@C>hy(XutC|Wui>U9a4KEf -z3k-}$IEk-a9X!|9v0XnK(cy!GsSCsd8ice+k_Hid>d-K=3(zo|PANzNhzre~sF6Q6 -zgh0Fz+TD(j-^pe+2t6l)48$481E9_hp5-hb&;SH1r(*#V*VhW|n-E{5dONh%{;ly` -zKyTo{P2=Gv!0A*Kh^4+!V64I;u<-~H@{T`3VULbD;~`q*xtU&T8qCKtdHV-Uh)2G3 -z5e_GD!cLqt5T`A~aU|+bpK89`c^T&Em)z(aH&$8mQZM17<4y!+5go94`4%i#!|12G -zxlXCvIo&MP&9U>$p?)a1)JtF}BC1Tp3p+{v=2)Jc -zIAC{N4cndOB#wEAhARQrXYh|lga5pXdJzt%!5VDaV4&)CRE7MGhx(-G1~9g>P3@%i -z+fOR&)aQniFBzy$w@PZ#_~LBLNi^7rV=m&r+3Eb`3=9sA;OwibOPH4q^1a^k0d}3_ -ziyy_%fp`WF5AYD5xQPA5q6LU659+8Rx(_h>9t^D(Tc(E?h$D>-W30p2itpSfa#9EF -z)Dd~-+3NG+1fGSmU7E;0p2f2e1`}a>9#$a+-67uETXe@o7%T*{7|hWxL%+XUsw&j#&;$wa*Z1>q-@flC?agJA5$EAW%8A9N8LoW#FO#H)7VQwwp> -zvIMJ@$~v9TVD!d*6Sdw#d~PE2hh&Xl%6S8Z23DN)PU7V|Qaiy+zW{RuY`x?G7#$l> -z7RZe(`N!nVc4EJQ_ykl87xAfw_*@Qlici1uL15vV54d}$;U>!kidG`$40x6zDb}=+ZbXr%Q&l -zSyl}`4G-7}qmy{aWn9tK>AV0_d;8hx;T~e!#S9&>)o}w(;bm2MC&6^~55#_}F~AF7 -ziU$_KY+nVMq?7ukT3ouq?F-+BXoNdx#Ho#HTW^ydmo2I_Mi -z^{I#YATQ%L3vs|ntnx9zP*0kHL&{33u22cG;VDfjW@YT!wBiH|4O -z08hHpd_1`pc#?!;NhI3V0Z-D40sVj4iPcVGjf+@|cyitJL2{#Kh#T*A4RPRmz=6Z; -zN5Afo<;ru#j=S@q$L|1voW#2@Anw;Z^yt7<5VRoBPAqy_3hmNK&~EBdDKyI_LbHqK -zOQBVFLcfPdn;{*50tx+h_*VpA<`&iQTEx>Y)9Yd4?&HL&`xVcyUPUK@4)r0oflYv~ -zkC0wPqAwQdRkV6YuX3w~^eS34q*u8YaC+5;|5dO0$VIR|y~_Ph&po}$L~MfP^Ym}H -zl>8NFz&D+$ZIQM~JEdgtCD6HWJ9W@S9Mr-1**1F4s7w4C`;7zruz`S+TD@xz=A!-u -zDK5$oi~hda8>sbKd;Z2hzW;8`#9$9`0C|Y&7&!IuPzRFF-36gqGl{>s9-h-NP!*eN -z?;^WCy{^kdO#;1cP1k5ju_hio&NT5069~n#;Mm+EX4`sN#C7n3X@h{x@dpnwxXSvo -zCT5|DP{`v=ry@+0qA;=DWLs^qy=1p-Ag|ozJiTx)00T(GJxKJ7-WGAIaE8?A2m-g@ -z41WZk{IbVr4+3MVfL$;Iy-u;c0Z=S5(bn{|E_xPwLDes@sC86tVlr74pJjv5u87&nMorR(>} -zO2n12lcIk~tHfJn$eZqLwHxfUg|7z|KF|>r8f5zNZry-j;QK!! -zeETFmPq~@Tsj_UUIS<+e;3at^0-g4@LGX-T@IH}(>xv&Og9D`oF*qlY*CbF#uC!1y -z>qO4-0%tD2H1|ZD166_jzTKfAfAcy8*jh>-wiIGqX4y6tfW~_~DC;Xtt%&?pGPF*> -zQqZQi*v1W`LaO`Ia^=c8F)S7K@~`cAd}yY$@I>561Zf|b!~wyz9ef|KwI__N;&Wk~ -zbgiW4_t*JZsI5-hR96STHQ?L?*PhX?W&A^9$#arAzgE$?F|ZOsmTONKm+bytQkq*@ -z=<)UAZCwYy&QDhvu3N@0U4K1}Z4s^|^!5(E-C=C-g9svk2;gue0_We~ey#XWSgYT; -zvF;8@>G~m(yo*4Ib(Ir!d3=QXg%%+w8d!vi6W>9g-Z~hWgPaXS_KkJ75MB_LuHu7- -z-3$SJK~hH)P!&zV8nMby&VM3XC&COGam~1?1%v_jYZaH8&-pHGd@b>aF!}oLtFhLw -z(vDtnAJ}S23$IW4p@67hjPuMh=CkJ{cALU!o6K>ExP`APm+CZuS?GcMDvo*j~9_7+R4`~5kNNU3Qo9ALCqQYs*Y~e*%xgfFRDG1LKS!^pA|``e@z>hU^3 -z3*S|40dTHt{<#qHZU9PtvQ7X^d&)oeWbd1;l7av0sTqOQE?hrn93}~+a@H? -z^%l)>J19eNplQ_7&+uWWEkrq5gY{f6%e=)`&~VJ3XFfR(f-}Ju)k3|+wGeb4GJ?hZ%he02E2@fNlUOr;2`)UeCTj=(oEXkHcTcEQpN$qB -zB>9CiWbGY<9G%}kEjvdcpeTgMSP;E!e&jf~%G~CM63GevCB>zM;7}q@+A{f5)E|bw -z3J@%wx}ZhC#njYM3rTCwQmKJhSU{!hlf2F=uqiU@8tw1#4~a^18yjcZ9Rv0}%(4uN -zDU4bF@q6w9A_c*=gL~w -znI(C}v=cK?Nf5Kfq}c#YUX~qLoAgw(iAX}qE|rK~U@WJvS}2vaAri{5`I03QJr}eC -zSM#rbk_Dx}dP4tQ(!FIaE_-oz@mq797qlNweZIXtzv!lcdQ?0fD;y8{>;mJ53KO-U -zO<*F_79xRt-VYW@C87cz0+@owhMW%%F&!t-F=0Rz|GdafB)W(s*D?$UmREJlSI_mk#H!otV6bAJsx9EyiTya~Zt_mzju>4=@v -z{bhnq`4e|a2@jzKwZjFA5{0hdC-)eB1CRZ6kn9~a8J+A~mlCL1_!19KY~cxA#=B0Q -z2p6pd9un(pnFgo06!y{c4AflkvMit$tfl5;QS-nBy|ufe)AQ7X0n< -z23bMdREw=`3Vc*-+a|kOz5!tnDLf@ -zC-(3pI^(-qZ%|`ZkU=dT7+B#^OWbE}1G?RVWanTD^o_e{P{K-Jacs5cUA%QJA@6Nq -zmwEKL?_mg@vaEscA?9}fP38J$IT>-yt;NG$g%eJY<{I|LK|Akz;0UW- -z2>wd~;%lbpQo@X#r%0wdmWQI8~piX(^+iuZYOVFa+lBXDUz0DE#f?>ik&XyART -z)rIOm4Yk@JfSDQ@F^*L1tQ>u?dX$TpUa0%6izjmOgk9aj -zcBpF)ZU0^k{y(6>tBzLW&BGf9t$F{Jagl6rNf`K(@^l2q8f2zl!2TStR0`vh7cOkH -zgo3Z{1HEPIEVdwbr@t5$Mq3cw>2D!s8;Cg&cN7%Uox85tdGMOUwZ#b$?)YZ+#iP8W4t@i8KiIBWOWIqFr%8Gll* -zpKRQ<2`FH_84Lt;*;Z?{IfMOHOQ?s}0jL>&2nP1FI)a_XGnhB><^$~|K^{b`b}O)~ -zT4?8eqvJWjr|JBo;D>!O5d25CPP7~E=y=~*A>N}_2!5n!o1+$8ynsv5iE=0%(Ob41 -zPOc)#*mi$F0)67{GyeE^J2Br$Jc6vo^UqG_;bF-id2xyO!yz5rv3BNlHE~0o%)`*f -zbv0~FUXr394(_$m-O>2?vXh`s^XPVPUuofyUkAWOO(KQ}7cf6HSJaC{gmFPl=veED -z;OFTQa?wL&Q_Fsio}Ben5JV*1jA2CO0;qsAJ1Ea|t8v-9CGH=1zTZ@*`Z2NfvkV -z;2|C^ayR0uq-%N~ZQyV4BOk(ktvB(py5hp{sX9InqG3F^bqzh>ty~lKxGXGMX0|-q -zSfw*ot*b4r3Po~RYDxeealGXyjEaU$7Zt{n%=>Eg8EIHJ -zTyHxr!|;hNwG`sjUj5+FfkC@RE!)em(HI^w8+5`3|K -zPQ~%-UUI;jSAAzMgU49ZQg#Z4$3rk>qD);9HA1*NXOjSU>N -zQ)8;+))IKC68iD8*UrOax5NHDR_C}IwqYzT+1*#!tzM$88J|=;xizWm)v~7$lXl@) -z!k0m_wqF*@u<|5zN20n?(h^WJKGfP5e7W<6yd>tt2mHKN(D;JR?)nYKM{1d1^SsY; -z7VLYsSqtykhwR3?E?a~%*ng48nU1xeQ#;dn++RNTS76~qsEi;|QFKq3orlIi`25C3 -z;eapfh0Uans9a;v_ybt>%|TK-$V^nbAC=;=mD`2*MB~wlBXgg%G2_+XK!r~lNQzAr -zxvK&IAcSx+@Dv91gvQSiWQ$R2G_HUTfLF3cx9kmgw>qpI1g(-R9`x9zyMlvV8CX}Z -z+LgiM{_?rMfZcQ_Z-!H?Xtcqjnt<2Ihg!)dvJqrsTSeLKrZLPd6*TSE2f%3;JeCkn -z9`jxuRNa{5co)X|LtbqMx%T_wKUF1Db*;Da&h@}LBV)tS6HDX8jJGu{*^Jg`m>%wv -zUiBr_d~ok)|0bO^C#39)=y1r$0QirV<~)2SP#lp^=# -z9_RLrus`xES~3S~j)Yx2GPWgMojnt-aiyyBK&6Ds|5W8H&5}pK`ERFCo|JhZDF1TM -zKz}g!+h2$;{Jn<^axYSQzTtV!Xa{Z(G2Q_WF<9@$-QXoS%KgYplW~x}>%O?Wfkzh( -z8hHNtE$Xt$qEh`wlhNsK%Il?ZF&ejVaNnucgkQ^`zfYnS{ZDD0G=Tm4paX64b_yaS) -z%(Cs9Wl;WPenSOe_Cd#hyVHh(pDa@%9UD4yDhOON14wbu}yu{FlDm! -zbdaiXG6bFJas)x>VWw!73b?sS0afiQBTf6?g)Gzljqum7ax$lzY;!+_jAeO)WY!>H -zQ4I@(N%^z!$Yh{9*ue+tgOfi50Rrw-zaR^Y0j`+C-U1_wFNV#DC);|$F2s?EmyM68 -zxeJaGrm1M9sGRS=_Hzzq^S
lLR2~1-E8{P^@{;qv(B&Vyl#`|{P1DdF -z;4~fo)yr8%4BLD2MlLW9oi@HgTQjBrGno|D@|5Sa723J$2z`mrTC-oNv0mseY}{L}Gp*O)lym*dn#i-R -zqeJUk{Ojvpu5Z!RPBYXTO|G40xc4CSm}xGj7|QPA4j13^8GEP{$|+a!I+i$ -z2lyQRUdPA%iWYYI_sRE%^fYKQSDVlG@0H^(sBW+ox0WmUn))wvc?&!1uGZJx$mXOa -z$4g_p)aB^%+G!dNf|md4_%DbKfs=pYPR6(9$}e=amXLZ&FfI$=fehSNZjePJSr>?wiZi%NRdB;9u%0V5Vz)#gn>TLDz{OskTp*AmU#g-WjC`Y;ewVvQdZBP=LxdH}2a8Wdq>ZG1*qK-EIr$J`sMI~=X~ -z$8rC6(sk>jG}52_%4aD5{<_XwzV02Xt=cREkaA%tPCTDD`JUfpgRRJsc4^ -zI~8p0EbP$$G+=VC?a@&ZGk%;gwvzo+^GOco+>rx)&@uDnw?I{d`LGS{;#)*o6PD>J -zjH8F9Ma>OT7W)k`D=U<>=m;!uOV6*=vfEutl3iny*XrM~+d<=@URZ6C7N3`Ib-4~E -zKfU^ykq7kSEYnOj$D;gmhcViQCCQ7E*}uN7HqRM~a|6k5ocZ|?y7hLA6k&XC=-9%j -zoWt=sF+YJ31P{5T>Y02~YQ||}RZyw8X7~HU_wJbakL!kSCQL2U7Gg#QZr(ino&G_V -z(rJ3z#PR8YxU;x{don;a4cv1hpQ2a;IpF?xSEFAN?ByCc|syCI#dFe$z1siDER -z%74Oy8w4A+4Eyr3+i&?TjI -zDH(q6&3qUuLrKM0PTK>qETGC*5O56cM(RC|MVGni+p)9(q -z@6tvky6ec&&sSvD;aayY6>s^*BVJ1Eg`4UosU16FdGzCcw6BG0IRX=WA)9c9+=wU8t|ejejcdHi<9cYYcidn -zwCXKqGS*t;JBIkxuJLp?4EDTslqj|zq2O$r>SP`h9Jrkzb(^g3o5WBs6XHOYmR$MC -z-HccKpy0fEYO{RDglPl^gCmZI+%~PZ(nXr5rZy{G6bpw0M-q>{Wis`l*!feN(~^~& -zl!@ZuNFs;)lgZ>qGgFQKk>cT$;7B5e{DW!gj;ds~_bRxfFD<36(kkuIpr)t`LzNE@ -zB-JBmn=eRyAC_U2@6gMpH*v`bJe%@sW&D1kC?ox*ndJe^3((EVQ)cHy=cgE!r{rbi -z55>ZcsicJwln%=ir-I-D_6iSZ%;U$CVwzBBf&6VtwL#Ne(&e6Vz_!DY-Z41>uVV>= -zStd##l*2r!j$NBOg21{kv?&TqEx2y1GP|piuO7&{>dv~_l6Cd_ -zEV-PMKH@DQ{>fe+=I9rvOgN9l+cwTz3(J}2nw_}`Z{%V%EQSWG^nyp8cpf{PSs*!5 -zKqrvoAuT9?i2g}>?9_^L9ls@kCCf)(lBrx5wtIe>zDq16 -zMcy#%p+_#c@Udn@rTv`HpH_~{bIv6dCZ+N=5EBS|#8P2oAK#KbyComIB}ALfc&w{D -zmUL5-2c-+)eZTed`_?e>TXeJa%ncO=Yx)gKx)3r6TC4hG%ZN#J7(YO;`5$49{-pi; -zhBDI(3nGqK^6X?Y&69K*$7oDi;ea%Q=FZJ)w2Tlfk^f_;tv4Kv|NK@`&2ik@s;xOx -z^G*bkRtqfMOK(!@4ePEYr4G!F&XG?T#&4<_GzypD)%@$eAGg5&=|5?f;hgL0qAC+K -z-5>kiKW+~3f7)vz3w0!?jfB2E^QAux+V()?gs8~;sK`Z8%Re6% -zIRX!?FCDhko^DUFBTZz?tjP3RK6nwfifLi51~9m?sMgZxBFDYHH#$Mtp*O~t~7z@T)$rPHjOcnT|TwxV%Dn0k5u!*>iFNLk}J^Q7% -z%$Q&}kVg5+&lO1qTHt2n#uY+xqbiHi^rigf1=TP0J>Me}65^Cy{;**?c+mxe5PTTt -zs->_AE6hWVPntl#SD<73(Q#A`$l;Z*m4j!MH_(sDA3CMBuU{zzu&2I6{|=UoowU0J -z-@&$BEMG4D^CH%g$sKFWEV*Nxd5#Vi!Iqg}->b#1+H_%gnzqh-S6}MNg0`r1=&bOe -z^+{{aCwHu=0b*!Bn35f0mals*8(Q>r=4xH3YmqYls%e0xQ0FMpi6MilbdE7PF*^FF -z&Jk~Pr0c{9{hDybV|2{ZiL-iL&~+ZV<9M)-BS~<;ba)a#tZG`{OJn^*uN*vvc~i(g -zbk|xWIHGmynZ^;KGY!-@vWSzBzeBz3($DLaW?v!Yk#4L!bjP|=Ah7%$^NcW9Y^}3g&^aFTcO)4*QR@|Rp3V{D -z?}#&!{eu8AggK~D`FV|O+STN -zlVF_G=z$=ZmHG>DY_oq^aurN~3Il`(WW~hOU!wG~ntpo|ZJen?wl@{(o+z)enma3D -zMQ`V|+GBUEQ?Jz@>$YCT@`Zb{7p@6m>uY#o>Fp-dM_f({9Eejk`WHWoZTk17QamDI -zk>B|SFK+zv8{kTgi6)7%wH4f1FqVn0iHZVm@@5WtVJs_Y1sm62O!5=w8N_rgb`NuD -zuB|W}JcC)f$WQrI2c}U(2i9y0d$WlSV1x*Ttnd=A&mQOi!ynFLr8f{u#zD^tXp0Hl@J0jP>p?6jTMxicq}ht!{Py8 -zw}Max>&;Usf1^~yVE*7OL+*nzn4rcw)mQ=ZfEvlm*o!ijO1)5q62Mc*nm@`>St$}(E)~%^B9?#)j@e6pX2wt3P>lZ+*8lzJ_ -z)Z|480glv%(Pi`aES2Z@TVLyyUyC{2$5LwD-`kqgyA&Hgn>rU2D5)5WDa$EvaD@7Z|qy4$+iY&m3&H`jy8)CH7oi}c|sD>pgcM)oB=P^sAYuWLg8 -zjD5xnQI22taYW-CV%#ba^;h#jS3&t)hL)tQ9>rLIibaFs?xdJ?Ol+Ktk7RC)L_vJRdO4R8T0kpHA6 -zl4`*%rTLo<<2+=Yc9|{)L^@U+HZcd!d8dZuC(zLoddc7Nf$(`1dRPvz1}JXG|AZ>4)%sLW>4`3@IjHu;E$gm8At2+{9R!PG -zbdD+E;y|Mqs&hOUg#1RSi5g}R;-ooE=#GtD^-)&GG|@4Uj1?I1w(bzK^8;bOa<;v~ -zz`kk+0|-Pu3tVF@?Ueo}Ws_%|S$f3-Dm2aut9+ecHB@Wme`q>BxodsU-1*6E>t_M9 -zZY>S~FtIm$F!V2r5|ea}bPkKm`yQ~!za?0XUGt^-RSy<~5mZE$ltwSy%2Y}la7D@i -z97+zsQSF6;eFr$|j8Nz@3B9BSwUd1LPx>X8A<>TC-H!4vhE|*B7N#p7LaQm8U9sw6?`~*$*o>hHjzUabDTnpa+)tQiGMN8&I+EC>*)HiQ_&Xz=^a&SSStdWrZ?h9HFTKsaLQdFK!Xeu-={P -zb(+3QbyCoI;UeqRi3sl;AeiQ)G1PJ%X -zgEFAazBLH86S$LTjWXiwG&6hu3@(7?>W7%2b-W?e;@TECXo6YF)zh}SOUcpKL0zfT -z^&hzogR0UrDdQVt(HM4Z&6|1T1qQqcMt_v1q_RiOg5oSE-f@at@V#RK6OId0q_`%4 -zN)Y4x+A+)|?0|?Mwn?x<=%thXNZ`&oFYL#~%t1|uZ+X{odD%7R4s=*A&QvH%uCY(s -z+-KwvSlj?rSd5ORy@gIX+*WNCHnD-A0h~Tcp;||{j`al@;3s0)uIgp=&gykg;(PuMm02%4kx^PyMu=|Gc -zCveKx7ekQdzcFZ|R@y~=@Fh9U7jUX#P~L?sT$#vQk=toqp|pFd%dM*epPd%ne;Bt6pLTnggy>jvo$-&E)l_~uV}taFH& -zowy>kHOhHkvbS2C>5}y#d&gH_9p-dmZ9LZ9E}H&k@Mez$Y;sk#&%P8jR90lpA?B)}GHsoXnM@1Jj!;WpkR@-!e!({H -zuohN94Ijp7E%SaC-WS?@T8!hTn`?_Tk`e6zyCulu8UW`)GN$>yu&G)*isLZNFQR0V5G;hO$2`^X$>0R9A!C#hQC=~MBG^*$;}a%MNSPV$g;nrRl!4{( -zIWvdk%zRKyW67~z2*~IVBzuE(lK`@<2YZ9bat>5b42KtLvH1lp>%jS$4hS3ftT0^L#=jEzHI8H0cP$xj#LJ@)6-cn -zq(X1=bj}(gpFuBJa|(vMb2PBBAinez?t8r^J=lO2T*JHs^dT22sm3RazSKz09 -zVLzIB_M*}caQ~o&JA$FqDixUIUpFYUWu~;gy)!Xw@+Uf#cVnNz!5*YxOQmJ`V_X81 -zpN30*hP*53XHC0&3a5bD*cE7B#L5gch{txdAXM9;}A+`}T+~2=vHtpzra|C-C -zokBlydqdB<2|Pvsg|aptih9acKv#h>@;ce)pelGS3MoFLc}Z}9M!@?Yc)e+Uyq8|N -z?UA8B!5M6@Y{dWpd8X+`*@a+nmMIUWfV^aw@{Y>M%NL_9Vn6@FN|KL1j(UYHH1UlH!p7<33Ij6ez@33CrC__!A6h5u_i75>FY-=rK6KXnpM|4O -zF4J7>^o}1ZH#j$8_3W<-+_+-m#5Q$oF^;duRoKM|uSf^@hwb4XxmxwTOCJlX;UBry -zN0STf)?5BL$GWVA{-qEAC;2-@kHqM;c`W-g2KVBaf3O3er{u~GrMba!JY!8@6BMYs -zGw#Apm@uDxM6s)-y(rL;HUZey;;~%Jc##;tr!Zylu_;v@g}TmTcdVaTav+{qOlIt+ -zNc(w4Sw=?q$H=>QTKR4G0YDQ7Rt(SlK7~J}?C4OwXmErtB?*m$konSwY60&e13QJN -zsxpZE_D~+_UlcV(-maECMR|vp%_`uRP9kY&mCiAk;GZJNJWATBCLJTattKt@W@Wp{ -zznZMmniav%aooR3U#JtpcNf)wW;JiBj1Co=#hqi1*hUL?8*Rgd -z@6X#FQgxAR%@;cnTe#t$edSf{5R}OZr2G+BmTb_r6D*q#<^oZUW5yn>cOR@eRe$2H -zb(x>=(Rtf)!Ilx%?L2t%l=~%+DWM%KY*|gFLS5R^IiSow#(A@yJg{PZieJ;5u#5j~C80bwDE|k(Bvb{Dpivg -z=*XeW*I{CP?kTI6Znc%^ntN&tJ%wZz6*j}V4QSD -zE&k6Pju2kC^is$AN{{r<2F&#X-yOwenC^_R-%$b5st!_~<#n<&UB-=0_E5X>Q2Y90 -zya;7^hv^75AQ)S`J0Z@br%^cb4%$ayc(f7ErQnTsow`aOY;i%9WOKb&V;rOJ`cj -z64_PGzb%C9zCtS{>e0~^S|w5MUMg+xz%JJ8Tkiz@Xx(UlM=+!OZeKXCFWk9ATw)Mc -zY6bJbd3iaGetpYyQYR3!Oa}qB5PT^bwx42i55r9U5{R5{qg!;F<8ra2IB$t7dwH2k -z!4^3fn-++qO+R_}W--A+Sd3j9<|ocFh!dcUDTFvP#Y}^^LQ8iRg??Z}L#Gcch@|IU -z09R^owZ3ei)J4040}T}hInU$7QUhjcQAe3Uy67nmhtDQIK?pZRo{J9z{csOf=1`DL -znT64oUhhEuPpN?@vy94&V?Q)t{uLmRluxlAG$8pH=?^@YJe@wPrG@mdNK;XU{7DNI -zpO3v|C=VFe37`{uRh`c?EL65M&VYT-7`+qPwQQ^&8zYPj{iB9Qpkg9GL)EvBlcr(g -zBRvGBQ5FsIXoLJPE}5VVHrgQd=l4yLd-g*B_1iWiyx7LVzzj-Xw-wsGgh3~7!R$po -z9}Mbup+=r!Dx8&5a%1z>8o|7!%2cKQa3}V_RyuN)Z=M|j6<{_CqA__El$?bjkicNG -zUZvE3uWEW5ohed+fi2ZTKeR{>vXsP1^~>n9OTU&r`>^rHS=32=xfTU}s)ttW4R?e4 -z1NRp9ilbWhSUk`9X6_KKPNQ=s$| -zOC1FNs1pYS83bs_PYJu^D@I5!fgU_qn&fl%LKJ^wBfL;-2Bisd -z9C9<8f%Vg?tRne^uvGh~6z6bpIL!wGC{f-_M?c7#rl%FC_l{_O;td?2kIqG}*LZQQ -zufd&!h)N7UFLzH{kE~be)y}sb6prrf-Hx9=URZ!jp8=PYll1dBVO2eqpyKZ>OnIrc -z6-#JME30mmKT-K+&ipG!5H7!mA7u~yyx+>lQ##lx^ut?%zBvn_&d@gLgA0(to7vDd -z<$c(%Wba&)8+#8<-XC};s0~x=d2hQ==ktJ) -z{n^*f2Y$By8gOs_1;C$dldhs(PD9_o(Vg@}n=G|K&;khVk}OVF2LW*WD*0`~7#fS) -zv>&;R{YbER*NE8o@&Gri>~U3%+9z>Jg!=Sj0YKweDB -zE&}W(L})Fz-dgI~B^AL?zmlp^Zt9&V55#VqhDbY3i4oOqr~q -znJT4eB4RddwNgdW!x|wI^z3KQJce+)$2l7n4wm9HBu!P}MIG#%9xt%)f}7E) -zay;)t`9ZZ06Ylk)vq=8YSen^im~c3N`g|}hqGiTPOe3{Y@3jKKWC;Fs>R^2+zxcgU -z;-RY07Qv8cAB-0`O$keKQKvH_uS4vooffd`ZRw1Dfmv$O?J`#wLF7OOjml|ki=JLt -z;96PW+<=OnV!zT5D`_a6hz$VH__U|Rq#hhU$zJ$g(BErMsXY|z#4KLA(7$+}@+ClH -z4IaS7QK$?SHmKK`)gzd_a=jFLXqjCWfop><#>n-Q;Z#osiLKxnlwgm1Qt!k#LXQV3 -zOa@Ms_?Tm$nFZ%iRKr<>N7>;4wuVdPd>9FSF_ANMwa}P5(B_`U#?|QTman7h -zG;4KIwXqx*P%8YY^>g&{sKz1Y--VB22aVO{vSj%$x@Be|V(ehs#J%qusKJ7D>RoBS -z5q!ix<6YxrQ;PutbXUy}vEU>=kS6r-5Uk30PrkPoZm)a~=x>P^*x$N^VWTs_JO@i3 -z7|T;eukiI_p*_~jk<+NG2mh2X*oIVIP@9c~$=Vgu5bhUnz$#%bo#5G}iI9D=V#?lh*%(%5Hd -zOJ@OPz_aB9_r|FcF4I*83zc&cbDhDxI)a5dGg3y(+o+X? -zfQ*2jLmJ$2E{)JsMrdmzbjs{zFYe0yo676CoAZYL8t$G_imPN!_B>uml8p$W_MneQ -z3@OiPhRAX*Hnq|`xTN{ydYoJfnz^Kx8S4z?;PZiQILu!x1Qg+9#oJd1qjeL|sAW>H -zS~l}g5V^r_3^1b1G!(2%M!{@SGn-De+-VkvL0SdrOb!?fdAJd?2fde@V}Z-hR;KEXd?DZ0q^Jt}$x9iqab{Qcgz -zF>-r#pX{f!D{9+vYHKzcRvKd=F9ob8&`-4`qvR*M)%Gwa#x`$Q;RLR%?49UHCvP&A -z5994yh~}rBnscri=O(g$)4{mB>-o`vlM8jf>mM`Tbz}PO8&X1Z+Vz=PfsB+?po@a} -zg6lqYV4iRrQyVc)4kY?R<$xE*efU?}*^i#e=-AR|RvR6jA03sa#ZAdP|C$S;SZjRq^%~-MqH!lELBr_w*?MCO0oamtZ=-|nPhTj!|%TLJ) -znwbTvigCRU-tR#8!R6fcTnMU-4$9Nk)Q8qw4XuGNbeEwt>!JLt2XDpHd=h&4!JO#e -zrl{=v&}>&|_P-6shV0u`niZL!H8A_-(44Fo6%IMF8AF@aj6X3mR7?pH!+Q%l`O)S< -zI0Vs2lQ6J2cD&8yL_obekQ2QVZTe;cCl*ACF+PN?@+89jpTjA)c{-v)bD~3<@aE=G -zZPl2gW1Geu=?DGh+N==b49uO;VS67iRA%84ZR4*D^;uy;IG7*QI!7%lRX+!&iN`-y -zGks3ME@Op(J=~1L53DGsY2P>Qrv5?*2G*ZuD!YC;Ri73DjrjoU%YqpC#0uAL7=W?v -z=mp(o>llA=Vy{~01^m(Mc+#|F7S#^63Z2;G5sc2HZw6`waA7!Tx#n>{=1wY{HAxEz -z_mWw{lWBUfe;){=qoK-JgvAItp41})t`@((q(Fc=e0eSaZorVRsTwzN8aD8{2%3UZ -zgE!D)-$M9VAPmGiT~6Z?q=k^=oM+V@q`0Jxqxw-AJXPB)2QpUwC`?@6=@uS0+3FO)LmVYD-56XF~*X{Njbu^)EVRy_~bsCOULC;z^%%MpbHW|4u(Pf>&)5L -zgRb6)q=~*UNNK(dx)ITg16zL9D;yKT%M*)IF#o^gaj(w7tg`x4jau6h*f+; -zO+E4&T3lkIV~64VAQvYH+vxlt_E7A(*i*6B?OB2LtYCXqkUcBJo)x-LzcFNE*2ZTz -zzD3w0@**r_V5N^O6gP -z7Cn=b)fY2u1PM}ylW|}1aEq7Ufb=w?ETm9h6oL+P?J6jqYdLML7R1puaqbky62paI -zZExfjf!VUoGwqwO?+_>qY>eETP#Iaf8Eb1yh{iu|XF-|CFE -zmPUuUuhbGLf;cWOI{S@WTt_x(H9W|=sjKXiHL2_9EbFCP`)2JsRf&WF>%aCiMVAe8 -z4CpV+6~=T#hj&H?=WW~g%&qGkS$(N~Q!CopUszHfZR>~*xw)rWpwL^-e0#kKp1g{qhXjK&GY4l3VcCxs6dMX>2?qC)v}Z5{ -zXAQWTHY+Vl#bKo{*3g7aHIs2+9MqZTb%a1CU(vLpJE6jY4{%ey`&IcNx$CRatU&oT -ze)(n?;nK~odSo%u$X3{3F@UV>x;MXZ96ef>7R=r-l+&~!JO|J5n{MyE40NU2Q)nMz -z#tDV;*H%Z$1CDUmm&SootROuJzZs{#-sO%q8 -z`p{yphQ7yH>YjE9e$USozz%a5EBvq`(n|&xdHX0k<&P+Z3tu9h@&J6;O)dtZFxIze -z&26p}{#P^9!d>N&yNeR=h-*SwE(;~)8l+J|nh3Zu2B-k<1+Hd -znSD@USkv_VfxF+2gepP^uN!v((Kn<$FAZ(R17AP5m*%^&g;j?!-(!XF>2nis?*_(j -z5=IA3MhONfh?lS@vckXsJ)U1c3A{~fx?H6RawKn}I-b(tCMmkdFAZ(_t9i|>$Q_ZJ -z^SxuJJV>#qA8XDx*kW;5uvjUL)39y>6TWV|Cl5e)Ihb9xoqo!ig-y!BCQO=4UIEri -zcMhPlz^?USgT@hzM}!pekQ~bZMHrY0M1d_@wy-HiJI{{iY6)zrn;%-0XdndAD}Qfl -zq0K*uZB%txZ@I8pcCTE?-+bpg3 -zxg%*qN3euMCL08udLcvgr~KScQyr}@ZK{UIN^)_;=NHI|!AY>UntJxLhRHL{KzM^N -zB_cxF#>dHbmx53MZADDHuvRYNSm2AJPOmNPh3<VyJ)kyqOXo!Myi7y#BGMj>D|_ -z$3712bF~eqo1W+~ux`A*hvzxG_@So5eBA2I2R;(55#*w#79-M=rcEJr2}zt*llSm;+Ff8^h)=@fw(R_7? -z&Nlp4xDb+3d-vyKpa)_MI6Y`9ZOtrl|I?VV!@g7Dqz8(nXkQ%r)qQdO{W|>mb{GbA -zkQQfu>BZ9{Gj)pRG=W9+$9F;P#5K5wZjb*HEcTv(6{r$CqaxpKg??j+M)|8|y!XnJ -z+Ao0M%`@uYpbWxkgp{1|wiP~`{Kma__*ctY@A-N9)$&`u^xSVliTCmkmbd&O{a}=M -zFMf>t`n~w#S}}N>XrCY^rin9uE2e~DhvHZPLIVzl_Q2$jHX?VzD-%_%L5b={!-(2R -zm$B0EB(&!u2eyN%1DnGES4rnfar){$SSki5i}uN4;xsX3mN;{+m}wGM1kr$^^kP!J -z{|AQ*LvoelFRzAmw^DZ@aE&jP?yFFUa>?uPrbmc+AD}|da@0nmkH>DRbUh7Sv$0z% -zT`T18J!5xNx}IhD?yq$H89F^j;e91HRVx?;h!?&x_^&;-Z_{Mvj -zj{SJXxk?x<*b`z4u(X1=PM{+f)ZvKk*ZdlY;9bv}o1nM+Zd0~BAu@8fcL~N4L3;ww -zyFZ;Noq?~!zhdx+gLY=uI_H7-*nvu5|64qzGjzfZY1tXPSU1D4{X*|T?Vu$A%CaWs -z#j>a)7jZ^{%OBq4SmudmRc^LnS7|+mW0eRzZ5~;bOZ49YS*`_4^bB3f3{%PsI5d^X -z+S25J(&S#1$;S9_h>c5U_{Yad^flvvqY+;rK3$@3Z}%yCa*#baEG4;rN-{nCU{4OT -zCx@ma_e)8RD0Mwi8Kj{Mu7#CBI!2Dn${>T>07q73kO|V|`0&agfB8#Kd|YLancb+fX>P^Y6?JQ(YUB(~s;Fm262bXS!&A_=xB -z@El>}!Q6e?-B51*14k?B)6&}Mk{j@9&?m9zZ#68Ho$!Xy7|`59I@l_4MZOHL>YoE) -zv5Tqxg&Jt4ed>v{^B_&S{3#V%s7-JUbOeX2d1d#$z}@>J -z_h}>2397?LcyfPG>AOc<<$<(GLA}}eK?$1PYi?%eBxL6WMMei^$M&Y;C=U4Gf0o`k -zz~e?tI7^FuCj><%AH9@P&$?btc0~@)wnz5lF|$R!criFeOk`q;U0m@iF+U>W)X3nz -z5noRZ4vx6FA-ETwd<^!F*uOpaXZst}3G|`5*^4LzUpYtp(or;pOCC=oj)i@aVGbu2 -zt8U1pWxYl;hY*5uZY^5n)~G=Gr?TET3z_bAH0kkv8wyO$aL0Zvv7%J2{N7n<4~~|5 -z+TjfssuD`S+^{tuq4b6n=tpV{5&O1m^@mrs`av!NB{zr~(U~aYXKAZ-N*_NpEoH6N -zu@HEX*gi{@ZB%Fx*5xVFLcbbsG960Ld{DA`Pas+8*%4W-E7yWW@WFy?D}ptcyD%Im -zL5(&c`}x2pvI3uWL?R}0rkB@H^(K@Xo)|6ky%X$#k&&;eZKG;~H3-D;@<6_>q5e+N -z46oAi`gd;&bh%!I!64MKLk3>~rFbtWP@wRxg=p6Io;A12w7d5t?A{SrP0GIaSI1=w -zXq$A&+qT=9PfKcs1@>gP{EG};Sa4bfwGTZg+W%4Bd6ZYJOMvAKEz;hDN4XCkM5=0( -zJ~*JzTMhl?H8;jjNEp8(5b?^rf^gH13gzKQsP;Fl@qoV#`Op;@phru_PY7IdJz;o4 -z0xzK5n+cFW#-&1qf^n8c1pp7${4jlX!gLHkG^Pdo&Qh!o+s=XlmjIc8ttGfJz)JZv -zq4e%fQr1w?&Ox+w8(SRgM%UbE4sjSr@y;TpA{<=iUClVNs1F>O&qZ%nbAvYI`w4bc -zWwNaxXbZwqFu(T%37d9T)~xB=9S7>K&SU6oopBBH@&cgqX43Cga+=BRr*Z;oE%OeGV&k<8dPduftS)VJ@iW?qgVGD{=3cadng6X -z2{U{2D=^oY-m?MlTW*{cNdJL4uHshD2+%R)l|Q3Hgp+DmPx&O(+R$39Uvr1`$#~J;dTke`(bi$B)N0jblD6pUHn1dq?6o%R5d4(a+p6(CpR}co -z_OvIxbxC^wlk`3>>z>|>{EP-#E&aj&!{$h1BR&=u*oghXnl7yMDJ8DA6Y!om4$>sD*0iJCS*J=gkU6MA#p&kUn3yy<-=q4KR7bw79#{!s#6{EPVT_X7~;O$ -z<%iBW!h7*lHsxp|g|2aKE?F(86$}> -z?pR@n7%T;Irv5tKNyEF|6$WXIx5zl`y{LK0G|Dia9ixv$);97Dyyl$}bUE5cRA%>Z -zDFJ9H4bk|h`q-qm8_|c!aI>N0 -zNiQ(y3Bu^(r^K@*q -zn#972ybigLN60_|GvkF#k$D#AXs9va-jvGgNTTp+$H>%6=!t&xS&huRVvAhR6ATk6 -zJWjFkAQ^o_8N!f~&wB||bDlysBUIi-8c%-Zp8pXprcgv&4|DH8-oeu828_)024Nnq -zYan1gaM?K<4jT)h!gng@N$oTb -z$SV1PK7hU#W71(TBJq)f@q@mj0CYUuzWW$F$siS{fPLLGUu_sNUwfSSzG4+CM|{+Y -zNpKVCuv11V_}Nf3zj6WBkH8?H_~ve8+m_&ZJJ3}csnkNVke%&Dx1XW2OZt({koR;q -z6ycd1HuwIXYaCD2oL-M9?oihnGWoOSl2ssvW7b-gf_AE;n -z?tR!p;QyXF6^{U-mz7VuA#J6CmELEeCP|ay4%8Zz6FuWop}V>#NXcxJ$D+y9HXS?M -z4Mi=wPIm}!B5QV|S8oMi-u1}I5c8lB|7idz($U+#o2MJZxFH~mpiF+>!h%n=Ng2~v -z>Ut7dW|U%9Lb5Smc8aAEyxq*WfJnX2)bVn(QshxKF>@aQwpx+KrQ;ig4V@Kg; -z+&xDA7;eeX?);KSq4IvWNec>&O(ap-A}?N}g&%b9+KSJ+4dcwa(y0^Wt~0o$a8_7B -z+bjrQcW=v)p1Oc0x+34h!~qXW|LQ#8>3=!{G@hv{c%pC75|t6={fr}|?gnQrb+I&{ -zaXnz5(T!;2C?DvXUwyOit|~+`SS4(%p__|l24#_o=r;|1SvKSutSlqEo`Rb&_^f8l -z^cl=r8b)T3l^2WjBdNU*o^lNP{>;I2Ac94Dj0jc-umsfSGJS4KOKAStB+N}DkQ@02 -zMu*{hMLDbL{ZSe0;+bY#@RRlet>rCLokf02Ohv0aF5Ei)<{+X|AC*K(AWg4z3 -z_0vM1`SNejaq{0iva9W`)($JiWGg;Wm>_1@TrUX|XRJ$z#? -zyyVd`=%4Kiv?=O)w*aZi$Vp;|ZYliY3W33*+2g?{Q&EEgZxgw5+F -z;-)wf`*HOAuxePrJ;(?VUKJcqK_X4DJcYUNq-{ax+b6Vl6kkgjtYMg=MV -zCaDd&3h8KG@&QPxbg3q20&(NkD^N}rTgznMb!Q(Qi -zF)}yM(N8BL9nlfurd1mF1n8mimo_dse(4@<#13YRys5DnR#Sj68ylgwoP>VfNcxk= -ztP&zlXK;mJ9A=;bnx%s>eLt_lFppDm+T_1q@$uM5w60CwaHU5%iKk)DT^VOC-hrO8 -zK%kC~4azqaI`&cfjmrlwrv`%6SRO;7%T;pU~^TjyzUBd-_k1nkyis2 -zkh0rIg3mKN5Ur;`$kwUl3a$Gf{ZoZyIP2vhH$d3b_WVy9)=e9G`U)*LfU|Vl1?9~) -zN+cmIg`q5bq-VUaI!}oqI!82F+1A4@3g5!s=5!K!+Z;zQBzbZ&M|Vaajt|Yl4%)@w -z1kpZ3Oq?U8JTA_Bn8q5mFGPEinDT3J=C8OvIzY5Xiz#?{Mnwif3h96Nc1n7Re-w#- -zWYs4xQsWk@A2m+OP(& -zp~v-WaMEnc>tJ}@_Xcy59!07U&lSVZ*5 -z;J}F4lY=c0i#G(DB8s*I<9YxYY@%^c^M0LE+w#BU<>g+=slD*O#D$UgFoeX1y~SEf -zd_NQ&AA7Ipdu3nAsa5`${JiX|Ikn&XFL7ZJStElVj(B!*@X(074Z-mQ^Pq_1+k@>9 -zXZHpVh`4klxL?HG#^4aKHZcBy^TC$*hkio!!$$VBjQ5w?_V9?|lY_%ROVA5*x1bkx -zY)3B~*!zpLK6tMDH#{I(*F7+@tYhDL+^a}f`ZHoneHb!L=Q~28c7xhLx3EW3lfwdP -zbsJ%L;!C}@qo#u%Cg~`Fx?q0XN52txp2wliU(OxGedlo>$ETeq!G7po5?((`()^dc -zt}+T?ztGKdb)c8Rd}GWi*{ln+yhiyr_e8I9J=C{aj+~;niG`_7I^l)|@&9<~othW)U^ch`7#8=<&(cG81^?PZcR-8-YGCQ{m0ainI-A5{ -zu#W~CvVie>u+Yu|F(kl&yYPb{`{SV(h_Oan#9$E20IvqOZ^S5krXXeph$~ED&OPKn -zcuZnuJ( -zx;9t})rDCZjPkG^Mm0IP_MP%`{R3|rX^5I;pfAlmodm!#xvg+1bXo(-`n#*#TdQK- -z^P5Hs!pV3pn)TxEL63C@=>**ISmTU%PhlUxlz&ay2V`*il#JUn7rpwM(y!ZM%ssYn0N*(B -zb0!XB1Q?Ql`v`8rWyK0o{J8*z8#gzc^T5m)*d=xE?L8MZ(6Cb*=n08IV)PJknAI_X -z8%?b?G#rO&ln!TTlur2sypsGbwfTKY -zj4~a;p}o`g< -zwJ{W>;xA+1Yvl!Aeh?GSCQkr5qQ!*604#+*HXtr#p*-l5-!*{;?NWZ%v<%bTtuLO3 -z>vaLO(2Rh6keagG-GyWmcXMh%@srZ?5S=K`(=ZLVtcRH^+@Y#u!@m=Z$mW1#Rp_0M -zd;^3&oA2gkU(SUw^|l327gnCZT)-6#P@A57d*p=gKqXc;R+!lOMj_ZeTrf{pk4XqF -z_E(LZ>k)1WFS}B6D-6+r>n~VN?#rzy&&2_mD-A%hWu!Q}jDQ@{ -z1Edn`1rjFcgg&Ln_W&*#{VjKMcP=d1U;RCRl$X$iT}nXqxc+ -zxJJ@Zcem|WuBGKR4fzM2!#Q-MrbT`9grUQPVk$aFc9y<1PH%6 -z3j0i@?rzuLb8FpqYyL^ASN2N<2OHV%p@=R0{%5x74l3c_fwYDS0|=^?l9OKsHtO&` -zIfi~N9XeR#IS}Rx0|=~U^!>v^@24O1(m~#?+1_>~-q$YnIx;J-{r_win~iqKv#DKl -zKbYF31oC!;{=c_No=(?6ru*78q||j#>P79E{on19f6Lp&e*1s63%HKwr;wWq1s9c! -zbkI$?)r|@Nh;)rEusud%vafMPZkDAU&wv9%!Udu2c<#ylxpcOlUQj$yo=oUJnWH}% -z0_&{xk^y)n{QYWH;7|I#mB6k4o;Y&~ieFbI&;*#sCjfF{+MJR#-vRRv-sPCzYx)rC -zee7H01saTHmS-&;Gd@XD%>M3?{65^SmDWWez(P#c{zVIEu?@jBiiNcNfuLA6Ct9uXfJ@MG+#4ad{ld$JWczGU~Y@rxzZsB>e#SA?Vh^Z1K>qu;rHD -zX|2%72|js_a)4wk)exM3!0#g}f1nYmq_l -z97exz!y-xw)RhgbN -z0#qth($e*QqZCOKAd=s#7rIr0!24LsMFA{;gkD<;AqGCHK=XTRU@1(`dTU66P}B<4 -z`d4FxQQF;T%`qsEbgy!gZ2wBKp)$qB@$g=OO&4+Mte0UKJ`P3TlLT%9s*ifS+hn2w -z<=0w)W)JxK2SBHA*yRbB+0K`Rc>tInl|x!I3k&|U_he4!*9Se$i&(`U@i_ORRXPpr -zq4|bqo@C%2>JJjoD1SuXfu9%(3;9La`KaeEY} -zvf7HJB3h&Pg2!DIo(q=Tu5i>p42R19t61vURo6@;j^l0AQB;V -z6B2>XALl8129q6q(Fw1T2$wufZ8KX156f?j3CCVS$6oMMVUr?1Pn?Gjzd!2cWA_nP -zNB4~#__BQnAY*^%KH6bn?|_!~iyVc07tGKod)4T0$kf#600@x7BexciF976n802u? -zHJFdn#@3iP0z%r_Nz>TidD-dyMq3H`pUT!Yx5|m-mcXHtY6C_xW*ZcAx)_hiNc*@4M$ws=x9vPR=)Aoo_L-ZR$h6}b`+r*FIgR}oKPY_#$4WuH6Ac$loWRU75ZYuHPY -z3ommWR=EBb9^w@GwixPQt^v#Zb&tdBPf9BrO0pf1w#OWgo*ROTk80m@+5>ZPW^0mB -z44+SJ8zF8@q_zza=Zz4TCW@;b662)N^P6FC;OFrc -zm?;1iQFG|!=UbPVckNg2u+yzrWdY$$O=m99}!RN*n7^H?H}gVb8j09O`^>3r`XV9)gWkV;QHu!eIwi{+F&o!|SfF=HwR%C(i@|w?E~n -z)h?j2z@`}mE1y{gK(s->rYI_)HJo!K~D`52z>h|T=)|%DSAc#r<4-_J# -zj?M~$^o0SJTBzICYpM+}sf5avw$A5y*@iklW%IS0Q9tz**@u-~%M04zt=V7cr*Zz7 -zs;D!`EuB!*yaj^;T!Gy?1JtnxT_v@O-)Q__pXL6iBLIa+=<}qpb&@CM#y`xp&|V(;W*R)U{)VLP<_=1`_@{6Z_yb3pNzlI&p|>&~m@JAR3u(u=;7|BavGuppsnLx>=JyCnn)S=({2wkL>s -zj|`2k>6YGaU0}vbX6O+;tes>05G8MF#f>G12&HLPmzlSRs)J@?xHd$Af9_mhZjPfI -z(i-G;(?IK$GswIs)SvG3APqQRy6>;fJ?SrAI)#FyR8DU>t3>VK#pEI{;h{g_fssu@ -ztza@<@{CUkoNR{78e)zJ<)cncu;3QhN1bbKZ7xXIJSlLC=?|j33`!63Qf*XDCF -zp^eBDiGWyu8mgE7$oY76DI|cH1xq18BSl@sicaJGGnxS%X7A|z_n07?o{`yeYV+a% -z2w#?a@cvo;r#Gy}{7XN*Ze8kM`Dv%s;$QpeHR~?F`cJP~r{NlV@uy-~xXAhnZ=DYd -z7GLip{2N#`x_U6n#K6NgbP2;p_TpkmFkuC3Qnmm8Hu -z*PL&2@FQ47=On$x8Ri&4<4nD~tJAj}U{{UVHU2~{a(sIQR9ls&{j8lhIr=}_$+ZhC -zwXS^ViUQ#V#}~`}$ni#eG2@P%2DcUiF- -zn5(qP^BRrc_@*FN8zpLMGqgRLd-CLaCr=tqKANkdK6|f|GZesw2===IPT8abe1Y|X -zNfQL}qU!&Ly*H1G>dN-TPmMs;L77CQN>db`QbE&*Ce8_yqM~WVS=(+Qo-%5u+i21z -zN^TGngG%3vW#XKEWyOPlk~U~}RHPd!wxU9!6%zqP4e=OR{t8%8PK_D6qa^3k8SDH4y=5j60y2t -z`IJH#1%gsWK3y6P=fn|7{73w|+>t2ScZ75HT^NvK+H4WwwmG4;#X+`}l;X??KihWg -ziU?KJc2u$<^HH&0=+>02QJDsFhoVxpp7QM}c-nW6V0t34(g`He;Y~-Mw}mF!ruMWw -zA8uP7YKxkQ5`{i%+cXL#3LVNYCnO$^(P80-Wg=T73ZXcaMttvTQ0{qWEF^&VY~Alb -z-T_sf=i~d{6=d%_XjZ}`p~oHN?FEdz;q!1xjTIa5aYKw|SiC8j7HruT97$uvM)_Nf -z5d~Ob2Ay`Ep&or7yRF|`R>pd%e{(f%Zq}>Vj?<$^vqC??UEJr3wbHFdW$4sO*BXme -z@durF+FEXWRP{W3Cb5gTYUyj3s@^PCiEs}KkABSana%M7I_<5|-eG8O!f$M+g%2(f -zUelcVW&1MDi&H{~w=FR!3%8jst0JYJ2+PjV0d!{>hfiI?`q3Z{1XB~Q#^4l74-KN@ -z;h3*#C5@UlDZau$g*50r#`k|qOM`{9G{~T(!93f6IhZ%2Y@rrgR4-dngl%ef+w)`qnJSfx($>p3{F;A>Rj3Ab2QE?||3@z|^hGGF -z;XYuDN||4lc%; -z(#_vAc!i^xkjcwqg<0xA5RM}lQoN_oA_~;}!ITIi8vMQU1tEA%U^|c$uoU+&G{&FK -z+e1OxT_8`pqy)H$)Y#Z4rnE}`@TTo^ljha0JRq((%@3hz7KG=UBo}H3T!ER6T -z->1e(?q}6TP!dwS~>xncf2Q(~4pwmAu1Rlw8n3UAsZ^ktj~ -ztEINfLnE|=zgSeG?R@<_Wop-ANN>{>E8l7^CP%bnf&FuFrZ-Iq40I<-G+v@2%Uzg;CjyiTKwP9eg*XM$h_PTQYMb3Z!yRv5C6S%FkE&iJ -zt*)0sYd{U}Rs(9Dx+-U@#@}~{2 -zG#^sLB=Qrk5YhA?h5wEi<&dAM3?RWHk}v`yAf%*4fY@iAAosZ>248`NvDvY49Cxzh -z32?&IG-%PVjlY5o&j@k+6<-|-gU{dgNyAV}Gp=MiE8h<3<#4ua3F+=~I`@S9res&% -zMk -z@lqagvuzOax{!4ZVHVS5#pdnLf2hfg5ZSE}FAmC!F+_>WYcQ;eZN##$h*czD!LjPc -zVTBhER$5A7rB;yh<=^|l<|Yc#mxM2_FU4O2mBmFh=4jFGZ4;6e0A$44L^=v&0Bkha -z?|gAtrBHfpzzIbC`b+tnFVPXMe<|Po(mP*NrJ++!M~q^P$b@bnJh_P}%uOaQcBfN~ -zzFw`0IV&IHTpPL6ijs}0C1*=c2W6E;l}u4d(Q?T~jccR2?Z~R -z-0G|u(FbCL!AeDIBw?4YKcbtAj4dsfkFvZ2zD9D_(5)FJ>p?7K!ZEzG38ej(n#^W? -zm@}uGvFsr~LMe6*;^E7=4L~H^(1>H>BHi)~Uk_QBe~tcLK{vt+WTVG-7;9>AmsGmg -zjH@;ukK*}Jfs*07NroAVby#n5MNy*PcP@AMeS90}K_wse!v^{XX6gY` -zGfX27@Mh^77&gEle%*dTk5 -z6LP^S$LlYA0)P1v#b2m~Ke;IWHn4SCyC&k`c(9kXkp@^A%N4x#>B9OpaS5csOABgI -zW%JwRU7-AN7wf^%1KuYM1Y1jgyqvpO{=+#e1vw5gIsf@ghic!;_7c^;KD+^Dh*B-_ -zubPJLcozIA5Cf_apKFzV-GJ6*LhlUHz*-1_y6dF{W7N)_AXL@@np{;Y?xQB9dY3! -zzj?0_Hs|div`_Mz|aef$-?!l-q5u&=i^BS*T_WpG*@hGAdJ7brtQ7WxB*;j0Czt>e0 -z68~kwmHOv%iwnRmUa6M4d^MI9HTe<2Th$Lj_fk`Pnf{A)?tylvJS;I(JJfhI(+i+!D%xp8DVT8es-&$JhuVU3=Q|>Mt3bDaDauf -z`ym**nn`{hjVDb$1HKz2jJTn$P%XbzSEvIyY-c3mOu;#s>Pd>AaOza|-k`B>@8zu|nsSxF}S -zGU>*3;L}Ee%wPx_pS%&I-){CT{+I*8dF{g5&y;vVRV_b`Z%}p@@E1BFoQcBSwWS)v+th=|Mi}R)?>!t>yNy@N$m{5wfvgr|FSb)SI_V=ok@R)uKQ~{uoyFWrejCfjERVV(nmH@TC?YtBGp)$}mLXN8gW7pPt -z=?Ft_iTZ#vs7^Xr8@}D!c2eb6D57__MCFZ;|6GR(-U}QMKByBctxCCVapAVK9=5~? -zo7HR!>CHosoh4}v4LOhq2dVPYK>6U4_26^va~$jCu(c1o^sorw3gAfrxMnXIE})f- -z^bXY}U)`f)T&DDl@D~H&k6VYpxpseX;N#pm;uY4wQbk8Q(|FGF@=>~-GqPGEjK5U! -zalHG_Sp^zboxgKmWd2R;UEaV*;M_O0P?I|NqfJ-Y*e6f<3NnJyzRK-%<%N4%rK(n$ -zU&{~c8g$TEtja<7b%xZUkG9`U>EETKVt)B3%nm-7l8=Y+ -zW@$WyZNVL~&qo(;|#|sHDRd%vT?R6WyP;BBNLr9KS7A-tI^j3&@6t#%N%9 -zi>2Qt?0in##*Nq06awke9^Ac~6JPww3ok-e-=^ggX0Bgouqg*yDMW38zEA!Ymhq9Z -z2k`?Nc5|E~xXh1R6U-eCEx7WP<9 -z$On*RRM~;ssLKaFIbKAfqdk)7Ir&F}t4?by!t;QpQa%ucl0G{*&@kG_@E-d`JRnPs -zpPoGOagoPZ+_y|+sxlrue?(QrSF7tG?E#^cuxj~@+pim(mN0b8iMMg!0Je|YY_Y{R -zRx{c8OI?w|l-vC^g}yZh<)K>RalR&A{EF(7=RJ^-bb)G>jmwtbV|qbT?5GAB83ky> -zj=H^?aYaF!fNjpUYJLuQq@XQ^ZELl~cLN4>Rjkf})U02h7Yvuj;t@>-lhTp4eK){> -zLD!d#H2Nw-Z>nGUsSdMiHz(LmDQz5Ut*~^+QdaWR&0(tiu)d(&8RThK -zO@oKN^SvGy5KWn_@m1v@$KiNZoO(6>%zQ2|-s${MNruwFdzJg=%kz}Pe0GrjeEqCg -zof&5K@03O~r6EjF(m4K;G_ZIUfBEBN+F^PhVun2{F3R#%yw%;5WsUNc-O>=R*x0^# -zyh?sdY=jz2mN>HV1DZ{}`XY%q)(BItI`(aLM4WkbkK|tv -zBF}*PB5kVGWM8=HYu{OLwgX&=vk^9g@9uD@Vys&VG=B_uO>0`QVpHQle};E_wDFj8 -zpC><}H?HEL$4m1 -zQu9(ZxHtZ-o4N}^a!WXNDPnrFaF#c+xs$^pq*G4Rcui}zytS=pD -zAxo!ZqrRj)opf&Ypy9zK8`VhtqkSR8#xhi+L$VDc@%(Uy6&J -z6ZPHQE=?MJY&8~vl9nrsJRF?8QJp&vgydlY(lzNyFlTiHzygDs8a -zO^u!_rR94}`eLK0%D$5=oY5Dmo*#`FQv2Zolf6(xRGHresUyUiNkWZ~L746w^FF)qDhgCI$ -zDie5v@z#wgh3TnPsr9wNCnF2ex2j%^bZ#rp@Bi$$gkixsgN*%O^v|+7EA@xM?cMN< -z*@+ljC0(n+g7BjOplRoxLPyb)3=2OnGu_AAsHrP(@!Q^aa}{7sF%=c2@7YF&kI37r -z>K8P_LKhAVS@E=AzmTDEhQ0^dk&2yhAj}uba0F+u%%|S0vt-@<;s6&LJD(4GQOE1% -zPg@fkf7dd9kY#yAY~G-(b=gZT-^Y*5&a!>_J4&>gVjWi+B`>_?)~a%U3xHa! -zw8RVW0s$=Z)}J4*o0%5HYey_3y0P%L7E`kG;1FkCpQdfWILav*t9Fc4#n|o1mp7~+ -z-np~9*`0(5g6iClL{0SOb%={4hFh=l=+#nrON^ZY-U31Kmi)ICL<4UKky%-Lf$Y)) -zLWUp)k~l5oGNflurhCpt;b;pH`M_z6v0#$M1*(WHM{CN-TAi>?i}!eQva%D_Pv7&- -zO+I{*<_gJ#tX3V;#+=!%+dG9`f=vxhJMh)^YJ$gAK&77$D5Prq3yuBQ5<88}AOR-*f4 -zy)f$*AD)^_t@uY)ZctKZtVa>VCN-~r9Z#5X`gjYWM!zd#6~uVLG#8~0>PGskjaK*?`Q}gHM*86P@WB%e+1@3?b)f5$r6r&2Z~i26$_PEIa4xwK@_Ypz9qaC>x~s@R54ni>|Za#v0EEPC0!=oQDwbB@}!?D4Z% -zz0%x;B}7X7>xRsG^B2URBn|$0NS3epE;e|@sA1p8hH;OD5y -z4?jn*AI*gBJ-v$)Cw@_^l9RrWCzD{CZ!S2Hd;jc3kxF^sKLMhE5F+6V={fM!VvRcX -zzAa)|u}V6Rj#Z_}oepaF5_uKQ+oi9+hP)*MhKT2H#P5!jzrG+W`hwr>OinT$%nu88 -zSF{H0b>SAGt{LDp?LXUBq;js#Ad?=-lMu=`}#72bq^EQZ{NOL;v -z)n -zP`##I6gIpMP!4{xgBPjv_uEG6t?ygba$tqe>mLlkA>SQWal?aoH$891+H-~vuxw}7ogB{1WASR&t}f__j$Is>VI0mTFkU;Hp0W4wJ$U?kd|w?) -z1a-g$x_|W;6qyD(*+Tn)`WyB|=3TILY0Tk%&x0&2J)s8m1$b}4;aC}nW%Vkt;DT#+ -zgD~|XIiyy=Vb6tv;GNN%`*0~mnn{Lv8q?r~IGd;Dny2AIacyJ0U{X6getu$j_O>H@ -zj7i#G?Ae|i#C4)W8rc$p@@(}4Sgivs -zgLIZiEySm>e#(a$6FbXd$PW$W{VZ$pUQ{P;=3?wCUw(V@_hW%fez99T`JvX#Q@|%9 -z^N%C{yr}X$R{ga!&C2Db(ziF?M8X&a)@89CdqE|?e?rmMY7E5d+Kf+Q}KHs%Q -zOaSu?)geg>f@Gp^6l#q@pvr_h^#vjITDC{;=(ok%Z@X!m$!dLJyW@o|9rOcCi&s9U -zfa17oqYapsvit9F_1|jB$zHU>RODK;RUUJVb{AA`1nPec+u@!za3ZbL(o=H*NBd*& -z0ENu=Tr_L}9QYP7vdk5q#;q36<9pqJ>n`sABb(9l?Mpn!@)8J#zd+2#i}~#FBhuTOM#zNjR2hrH51DB{G%q(;~%?jQ+D#?#A9bf*A>@- -z$ennI2Xv2}7@utTJ3I!|v+IYrW`rjmjXcY{u=URbZ86W!;u|@>f*WZF1qm -zl0GZlecp2P$t=fnEZd4oOBSqjEXd5=D}BL -zquo=UjEn59;3p*5q=j%pb&>Y4-tq|?xr-+r$L_yt@w{SbxQmy{9_a=g5vF|24itTB -zUJ#^~f9=H$(iA+$JV-C8p)Q#OEd3YzPGLED8dEU(*^#q2tTIBlEp9>_IV*qmxm-gN -z_Q}tE!;|!DMNX{!c|9!&ge{Z+@PT$KQCNqYaAq+45e##y@#r5JmQ-uxp=4N9tr3Sl -zqAkg1N*qejJW+PsPfXL%YuCdQ6?dxx)K9>Sb7-v&9d`VbdDwySkK585I)_91N5h1| -zltV!!4qTYXNaP_Fv?NW7p_JFV(zH2=1?u(Fozu%xo&wu>wO0P)eNa{sClnQ`L3o6V -zPRhMM=RiT?y@hJGLwDrj`4QUko8>pu1NFWH=;CwA -z*V{AF!1|Yw7`PUMWrvp6$8RflP`Zag)=%Hh(k({1;@bO%LJATmlpYR2x^U46Z{45s -z{vjn@@y+}2J90P#=@wl)KT=zMtNccYFWnqX-(ji=*7a}k-QJD^2W`qQHBug^Th~5b -z@y1`}3UAy8VhpE+1nLLIoDfTnlbl|_XBKEB+_3BHPE)IP1~StQ{$6E -zfM=mu-xdh)VREZXGgd)6nMQjtz88$Oi5+L&u#Tfh!_rdT9|FBVZ7hOTNHMpjDUj1( -z&QX8iuYz7lfe)!qv9f>$0g!Qo9^O0-qpPi -zd>Bq%ZOgQ_Y?bnKD%{fC4{^m`V;O?80vy*c5!Vz}%zNA}D@yt=n7waL;xr>ddhb!T -zHM7j{_FuF@LyN+AD;A)0AfDWIZqJYaI`a^AJQA?y|$6S^^? -zJL1K`t{EmKHPal1Cxm_R*xqxSl7BG`!zB7L=qO{VmQOqh!g26ev|rC{!REu;u=<6l -zin2R}k^Ec?=w)|aH1JC>)g8UVuX^3EXb`}b-r-Gf1y{0sfW)iK<#~}AmSBfnLN2R# -zULeDvU}--9LR!MyNxT?E`^P2KTFFX=Rn=O78PM(J!HSngWCnD5Dee(%xG-HlL0%@f -z3}S}L$Z>0d*tHdUO;Q -zN3eN#>(S^=+@}!W#v3zR`9QH(8+xbLHd-<>J}z)BATIYxaQytcE5}xS9B1_?d&t2- -zs5IE(2U{Rqh^0a6Z9ZbOZXQHbJi^*0>+XvN?pb5;+U2sh>*FBz_zH4US1YQH` -zzGXrooQ0B+;^6y(g3WG7TwO%Wxx4PhG=;8I#T`cpffmHf4=gRAFo-Mx7 -zO7xk(LcG>9rWIdrar)ZYmj`LZBTz%s=BK&0f@3G&T|d3=^az&jQX+L3>WS*3&TN%e -zGC0>v)`bqXm7=;L(j_+c3tGK(M;yzj7?YIKN~@}Gf0~xO^qO5aMNVg95U-|6(P1f{tTZuIc^TgcNY7I$*k>(%-^2rCF2bB0Hnpc9n -zHllQYQIZo+%8ul>SF5e#MhUO9@YXx&`N!8Vtze{ -z?m~I|k^gY=m!muTqd&dr;};-26DHSV`O_hfd2#KDjKWK!?)@tSI##z<~i0A7Y@|gbve8sk(dRPa(rgP&}J_u5R0$~U` -zc;*B2ukF(zEJL67!lu5{y{PZ}&~LFnh9#iBkl{o96o&e=E-ljx)Q`uOV0N%P>d_?J -zy*EF_Z2>IH(!1MNcwr5lIe-z(;|Z3TR@4|`Mc^ftB>I|QdA!#sEoAL{&NO)DCOTTB -zx=TbGCnqZH%FIcf%n6;$$%+|m`#V~GE4a!wL2DUQFK)p9AfZ0_-%APxPnkxDttZN0 -zy;cfG4OeS%u56Nhd@A>n@Rk{fYXj>C8u%T!HmMcDSdSv37)r)cGRh%{%X+0&=%w^Z -ztz>QmH1tZ@#F~!fN$KlJtglb-_I2yH(f5+TI1Zw2-+GMdpkg97fvs4glTj@XP%?%t -zDt4z{0)sTT9i}&rQWWKJF5imW1W?A7kWpa9Rb-U>DUZ}qDZ8wrYQ+%BBYK?_+zu~W -zJuWfh>)b=wM}Bz=sCfAn$L`PRN;wy?c15BgY*?rFzYM9xk8X(qjo=kk8%PdC(R%$ku1P -zZO9Y$DXR -zoDoN7;ERx6Y8|VkJ;Ih&$QgwbtpbGiAW?Xs;|tS@lr5i9HmI+@hYmMTJ6d-ywJYFr -z^_kyKGR#()=B(9o_SCM~0pqH+x@t8pgQHM|WHTQg#?p`aWRk2rD -z*2)YzX%)4JzFJeSy{(gU9G#}HHo-oAB)2E{J@x#oy8|`bRJ^9>)uA(Ec|D!blz-^= -zTb$|;B+OC@Bw9**%s5>$vzu+EskGj^QK_YT)Z!p=9J<+rQ|8)18AD1YefusBjhlw&gi$H5ub4rj1{K$I9N%vlQjI>NFV>CZ$af;cy_& -zQku8DG$VCcTIzb(KhCpMT%RDWOrLN-7QlK -z9}!d6DRfC;H+A{#=%Y&hWKV1Ugg(NI7CsbmP@^_obcp&NNDmR!KjhvN-?*5&o$r|! -zwF%vlHcJ8EC!{nmwtX{8{JhTbzGX{9m)s4G?Jg5y;=Xu_QxA(iLz} -z(i+zgn_O;89?u*AgMWuFm^+o%WKH)>PEMLU=b{EW;$USwn~&g4Pp}@n5|Zrpo_=r=xR+w -zuwz%=mWWWgR1*SJatJp7$+F -zPX)U#B29PI6KAU2<)u?GyoC`n!H?q3N|t_gN0{Y}tI#xscnpppxr}0(@DPMVRMBu| -z6BC+fWll`ZfRpK)JN=jkIk#<_9@=C*+QU_D!J}dTQQchCsJaQN{=0IDd`+Uc3(flb~Ap{e%> -z(Uc3FY-^$}Bt|uDNLHX8M#c18`w8m@gdSAK_R#4{B}*POM447P&)P$(W14({G?P$I -z+7jU$a}+K?E7M6|Fq=#^oW)!r%_N**RbK3C0^w67P!)5go{ltqY5z}k-stxDy9aaw -ze^*uc-c5MSOxUOFk9TqXSC8K~dl%znh-FLHVn6WU3=l3cj)&{iXo@F|-2U*4!5nbW -z>j|L^Wq!hIw+dEcKv#hkN&5EdNBZl#DmT`vTxbR(XN)(MvfG~FvjTJDw(?t~K)(NM -z{3nTdu~t5Lrf_;0SbBxZGk6C9+l`ydzdIgun@?pLPz=6=&v9lj -z3W!sNa7!LyZ(`&Jh`Ls0$dtP%23~sqp#s8PtVc?pd*?V$MQlUGd^|mwSP{!+)cN~A -z^jl)47Od2Yr*FX!QyC_ox^<`zU3=NH3rruU8MlIG5)hYD$Iqww>qRY(@&3j`GRU7o -zgLJSBAgLajCLBGZ@Hd<-NRSVlNvxQ!#G^}QVJ_$LGIgc4Mk9WYp_VsRYURLIdiR04 -z2#6I5Zi3f5+Q*+Hu4Loa_O4>Tvu+|d?b)ZDxKjL -zJ<2jYFYR+}+?Z2|V;J+ze;1}Gam?Ny!_lO@d$1E6P1-*)(>_p&rYjpyz3KgP -zxZlQ&d7LcfWeo-&mY2s8WAxCo -zR*%+d?dhZgD5VZ)T*oB&jgOv9_RLnyDBk>n!m%W?RXqC;hE<@0oOy`j8FFC;xJaKq -z#CBmW;xGfeq)#4Vi$gBVK!K|tD(R(jmdXTc0_RRpl~*RX6V&Xh!Lfv8-;n+L4+?u9 -zu(Ql`&{ixp6w;HRI#ZM2Nl>$IM@V_s16EHM)>N3-vCHCrXy0U`Uf6uqj4cPun7O`h -z<<-vU1IUG9|M+@t^l+S-LU4<+bynP^itAuJAHi>z->J({o3`fVy_}Wx@;3E)t5vP_ -z!U7@&-ss>{rs0wAuvatwX1K-T$O>tB?MC?*irrW5K0Wh#N{%-7Wm{P9Oq>ftkQlW` -zAiq!KPT*L|`MTWygpwA9MBKvW_euRu_}BrJJTvE9@zL@2JZ0vjE#>MhHDWz -z1+Wtg4S+58@kU49GH>pA>#03!<7Te?#nHFW=a=7@g!?y1l#0hM*nf5=1xs!K?s5~L+h7b*&?CqQ@)Z4wJiC9wn0Ko^7Xi4!9 -zja6^tyYR-%3%IJVub;w2*nJO~{Y9>-*Vm(9-}w*+hB};EH;3pOdKj~N_^ne3?YU-c -z?=5sY{O^45zyBY?|4GJu@QEgqjQrmQ1Mz6e|D6u_&vR9wUk{=Dulh;;R{>SD4I#Sb -z5RG{8M-|cmm#_&>FDtbL9A)m8ZwRo3jIdb+Y`()c -zX>Fm=wy3_g!MJpYr+NHuV&r`&K{E)!OaMgqa)0BR@COE4l-fSHjz$kgZqnYwfE**- -zy@649iTvG-wRf<|0cV#Q05DEM1J91wd;{Q860|v3jzL=t -zLeQNXpx>+#?%Wt#&j1OJZx95bkwK7d-xzydNuU)DNB|mnf1@ODOGyA4C2;eGzKR8w -ze_(+{Xq1~9O5p1hI7WhotP9s~94{ySABEos_2V5!P;K*X;Vo#2o)CPu(^oW$*h^EW>?d#&MYM!u&-UF0|-xDZxtaH -z`V$6om8ic}tU6u68*^|aY-kUE6ES0v$jluvtZsw{xBh6z{J~OW$#cZ!3upVLCamd={&nuut>a^F#YulAOlDmch;fwLH0zHxI=THUxt4CXX{)9N#FU -zG*%TsHj3l2G`5kdT0t9vCH$r__mqH#P=QFY0mnDJn)1WGp9oHBoaR|QsyQ>}Czt1k -zeTB-H*cdB0mo7*u(@PK9!XqBzlI+@VsUPNt+IX`s2fn0@{Af9$QF^u!N%Cg!k0E)m -zX!_bonu&D5bW7=q(0)E$VN83v!Vi=#F54;TV!#{I$OfkI5P?8D0)DhK0um6ACuvS& -z?h$dpO_wW6{JxgJca*uk4~p)_j=qULp%T7>+#l&%e1s-^#6~7gfz}11arSMU!tIQb -zv`Kfu0QX!^$)eJ3(9P+a)7jj8W)4uV_smr;nn&Uw%=9cCyCG8p|Qa`5BuuC -zr^~;)BhUHL`sb`{=Nnn;ld`N+m3H3wzAaRf+0d(L^EvL?G@KM+i~kRnLZ<6Sj>p>_ -zm40T(^zZAh&8%A+$Q_L{->-_xyf;?G&2NsMzp$Lw#g283(}HBEbH78E_+r4AZx)0%R!DNdknUsDO+yIo5-5;5{B+WgznHLZ;i+?967w9 -z=v3uN_1j0m$$L2D{Pr6gtoG+V8=1_aRBOSwq82|tUZ~k%O&vVb(VidBBf))+`zowG -zFh-Hx0NGc}o(_7lX*n@CzNT8{25ATX6{h$8G<>eOb0$jqkZg^ -zN|o^Gaqyn%s6Vb?4-KKOpl|>3ddZ%)5q0hYSa9}QUYaCKWLsc1Zi7(BYp#q5w(PbD -zldg|omp~w7A!cxFTqJJ0lxg6BtF5$&hBL3BQQtccN(b+-?N@cHG~i3A#oM1S?mTJ~ -z`+fWSWQ( -zA{%NMR0dQUR2*^yx*TXbQ-oEug)ddEu9;GMw2aSufuuxq3uyA9+OK7&jgqqcr8&4MJo1aC^=w4LuU2dems@d1KU`m-z -z{{02W{P4qbdePAjUdmZrsLy)S`C7jfPw2L$o3?FBPqn8GF4zb$#@nmVe-Yn!IK49c -zdP2V0QnT?&eKz0!yd$J?W3lScYCf>!bPstip13;R)>gTC8_)1Vg;(bxdWOeCxn&v` -zXf9^RavY}Gu*+Hz+fOY%dBasPOc-**9;(G)x95&MM2A3fKBJq`cqsqv4zB^_g=oDC -z9>%YCng-|iLm;8mo^+GyhK-pON*G#7X8emVRQLW=_w9G52&cS!uW>qywS(`q8^m=` -zNP{iW$0MX0h>ak^3zr`V!#|@m319hr7*{o-?fbYyy-TgjRtGqo|AiF_UO=80;7Zit -zMG=QHD=TXO4B3eRtS->+HHir8{4OgwC^1nNmq@H#48JvTiHtai^2q(qd=nwvyZsHA -z-G!FhE|s3|+5M)U-%ah%n_y9eP@VuLyoJpuY9&9ATW(2+WTMOCtsaw=z!F?(MlH^# -zyuxD{9`UDBM(W3J#O$Er4sWy`yE{+wx%w!E>i(C^S{tQfy(#OG#Xq5jZD7_MS& -zD+KGGe4!D7AnmE_3eu+q8|5F``tPE34$1h-54SOzFlefN;@aAI8Vm(m=|W?U&Xin~ -zvrd;f9n1qJYv0Y6=V|)wYESbJWXW^t_`JN0+jMe0l0D>y^g5Ec$m$*|m+OtAiRb@D -zBYzswXQUlhSZ_rc>0MJ4u~2QoYk&5n4&D~lqLEH{jo~ -z#D5qcuU(%!!3uHj^oFofXP`&=f!@eyoqek@O!F}V0{Je&I0)9jq4WS;!eWL|LM0ha -zOP)p(tC5qSkK-LK^s$Nd`HQou{Bk9m-oG!_3$N7jS|3<2RLCbA=geUgPK;S499_%ZCf+5C+G!3I2Qxooh;;(kb=R6)KlY8~#4T4J^!dE>816G-pBjU?B@VBnffV=4g;(itH=roZ?Oa@*oDJZ`Dgdwbyr+uUHn~2ymVnnb -zj=k)i*IrlmeIGP?qjl*^kt?F^vk|aYQ{UYWhPoitXOg;UMKIox>-+sckgNzlOT1?- -zO(n+R;&!}YhROPq8fRsK$D{RBpK(u8xh8R-#1(7YlNd#Al15(k&@{N&qjg?SNVPV9 -zy1(I5g1!vb(EZb*HcVX!93wk*|6pUuyu2z^{uJgujc~xgJ;hGFvAsEAx|sTbd(Zw?7dE -zIADYmVp_<6rRne%I<04Bxxd#@9;z73!VzWwMK%T}5yg6Wf2|2?nmyT(nI`+&o41!~ -zt&y*pcVZx@ZgC)C^%mE&iRr7dov&tPBme7P+Be1;xoQ$(BJ*0ABWT1TaF7|qp)HBW -zSYa+LJMk*wo9tUQS1CBS^W3#CedhOkI8Oc{%T`~{c#f)d!p$~t3(vSpS2?Kth3Zdt -z8bnns-4Vl4V&3+0MQ{}j!Q_uIDBPfzdtTsEoI}f-1_y)couBkeS^jd8LHe?ZFNj}U -zo@Mu#>{(-@gi8=&E?;WO+8h=Im($gKlaV?Ed@GfQaP;g8zWsF|B_R= -z`LhH09@%q}alKl{wkw;*tMc}gXWqzrCn)coa>v~NkV7x9>0kmMu$HcjQGSXwc1;xe -zEp{c7Tpm?}HPaPce&x$Z;%zI%`YQWKk!!ofxhqocdI8lsNa2}Rc&y|*ExhrgIf2m| -zvTHOB|7^#&|41B{Z7Q{=-YCv*fm~CZx;r`==5vrKM=7?BFknwQ%1%vzKv?&JAI<(Q1S^Lw4It-%D;%pk0QXQ{xGY?y*mKz^0+^*rbZG*59ey@C+ -z*}%WfAS$-f!>|FpRrOjy&6x(DCQjME-0Rg;sKQ9Q2*o{q4#i!B;;!{z@3#x%LmBpw -zH(peoehC~}M-S=um~^%s>JvM{iy<$y)|I~Q5Qr#Z1x)MtaJ`00g(R$+?DX{+>FdBJ -zf>+5awL;wuyWo#EFiD09h=+B&77}aeYck@6ZS7DNl6CQKj?{ -zw+5&5oL>-`5;2!(ImoBG!g8dnBz?mnBP2GlL@F#EXBKAj)3qg$8{lG;_SRM#4iRl6 -zp~dSY?)N$g6ObP;P<+yc^i7Z_KD9-E8D`KBzh%cz*q>U8)ypb^25OETutEOt)J3-E -z?Y8C5A115gwiO_J{rcl1@d}tRu;}RNRe&T+p);waZk&AO? -zt>w75-MH$Tb|R=+C+)8VGbbMMJK)U0+9_-G{HzTL>x({mw}~cI14)aC+r=G+*l8xa -zK~s&b`9R&tbm>pE?0E!@u)DTQTdjKCkb1UQBjnctbF>>OPM;MEYs=8GZhEsmL@0Tf -zvC%(!W58s4_R3ksdU5~5qUj_c@DICi`CItlA#WOJ$T$^*t$DE@q_!AK4^NTrUk3lr -z(lZ}A%RhXPKQYkn`V`Aw%2)j71biWTuyRn=+^7?hb{>zx>QL~^yS=;Kf+k4G7TBJPa8WZ(kK)J_bm#_k23Tv@*_J9PnPJULxYMn`%9*Z -zRSySZ>q@6VG*+!<^mCFrB79pr$>1`m;j-09gtcPlfKwK7P#WvhfUW7_ZM-ux9DBHS?!iCX-Z*r_^n5OQ0^Su~ws?C^bAHE#<3Y2#*gLtvoKz$=FG@*%jmR+8Y}zki53Fti~sL~+Nr -zAu!kz7=&0bgRzW4R?#gt1QvET^F1B*OqXrdj5_}kHD^mz>mxbH+v*A?Np+%H)$oPd -z^9jW4$HxyTnWSo-q)wc~StnsDW&NZTk8Sd*l20BucPF?~IcEXBD)v>gug1Zvn-f)u -ziR$b`O5a~vH~KgSNNpJNBs&uL2NhPqn*22$&??W-KN%-lj` -z4ELrdV0imYGpMj310V^zBGz3I?@k}d4p3gK^F)puu4||WN}Yb?CiddjyW9v&t@osl -z9O|fdRm7K6#8%^FQfV*PGQ{z%X^}zbh+?+~GwQTr_=A~6Dfu9*7%(cM{r~4Eq+iwM -zsCZ*pdO}8eVhL6lBlutJZ&}lv{`9pc)MH0+UXPQJu9Ly0b#cgj#bkH-v$1i`?DTk7 -z`mmDpq2+%LE~%JSl0FKTX48k4mpDDAA2gh9ZAgE{lRmnV{IjoLtT;aA8ilwI*A{*fqWP_${jfx -z?0oql5z8v#Ga};)bi%OKNO=gn%&`5BRkagk3ii%iIm -zipN->Yp!^s`P1P1kf`FW7TI~_FTvT^*5zcCI&cU_lSnmXOt=&bE>;@vy>iE2IWRvMjd -zC;IbY3;7YJq!g7}<>U7v-TlzTJKK|$KSz$Huers+K=4?p$bATfP -zqhAPHSA)KzmRzm$?qQiq+T7YtWlaJd3jU190eVTcCm^wMNOr#ge7A2392(BvQ{V@b -zcpyl`g$?HEgQi_Ff7j*wphTPH)~=SzzA9EpOPbRbJWa6tN2*5t56Fe_oYP{Hoto^acDC=a+jrR5Kl}FOSFkUao!9|Un)0u` -zX$t9-Wol>Al)TI7O5Qif&m)vzMBb&}Ki;|B<*XLP8Wf=V8@>n8vI1PEnw(qL&?+XN -z3ad#}&TKDSX|50MXpmY_NL!sH+SyMHw(@y~{OE2#?FdXped=P@LxP1V+0rj5~7W^$Y`KX(FyL -z_tDtz+rV+(z0O--;+gx|9&KSirQX0VLdb+Q#p*!A=%y?FN`0bA|NZ-S4g5b_1Alnx -z$stcBT9U@6SZ0nLIB58gr!28UV~54XT4D!}h#NfOsbT+p(bHZK2gW}2WbBZEN$I~^ -zuxLU0%i^FxBP?;TfG}3funZb*85H}1Wk}qM{{oItr|pNs`HA4otLXpE^7V#)5$4}c -z|E__5*TDa?HK0*F;1Z5;7S3W;1sw}gkMavX#tq{h+~v4`Xt}N&Tsba}+s_^2f}RNK -zrpH4y -zo5(HT-au&#_-b_hRD(DR4vW>uL0j8S&T^e%k!#j!a#h1sW6V>9Tt!a1hF<*#*QN@- -z3TL&(|6jWL{YzI>fqFTo=lnFw@RVBHv&%Sc05=NXA?!Pf8>LbAQ*i_qx#LE0Z5j>P -z$p!Y_uy-%udU4f#kQ=lj>Y3w$9;jlWx3vvo-#GR~2`E~SS;cgvUHlbW7RPE|yHnA# -z9&oP`oGeX(3cm-cKbBJszk6%~_m~SVXJt%KjA-}K;@9ExTqhrY{5pIlck&6s -zufr#;lMh6bS~`5Db@GYFufu0XCm)hM7gy;Z3KMl*az+rP;LprC|J+*N96 -zRs%pXR$IVcW(P8nJqv$khhZn;W~s86KjAXDQ%`wS^MnT8s^09|i+#=PYhvFZ_ATK4 -z$lX;b_!D|{2QUX%q;Bi##UJD{LZhQxs-Mur1DkuZZ!h*Wv#*JLg9zg|&OZoWH5a5x -zI!3lceAQe>`8!(eO_llB-VY%8hJAY9=>?w{=~mfJlwiK5lo-I9%kBNklhFs2u1NJ4Cy{a -z?!F(T|F4wKKd3LqT}MQpI)*`=%sHVPXL;_&W%0%NxiXPI<)O1bnd$d)I{8r?DueIW -zIV`#R;E{i4JDI6YJOBQbKe_pof9JqnvkyMy!`DVs*M3kx7v2}gmtJSP7cA3!?gabi -zeguQkqp*ZKKmGY&kh{s4En$VyRyd -zJYPLid{Vx0l9}?*IUk+v;KfB;?mETKdQUVzs}=KXLp~O>iz%bN985e`S(+M -zir16MPvs=MAhXYoVXVFUGD33n-&*_vd^-$6YuCI-K -zI=OdNzVtim2-5!Zm6ys$KVN46rLe!sao@cT?<|h{>Mcc4;NJnregN1FK>vjfvy%|8)6ap1*$s6B3arg-pF?%> -zW=3DV`|5$p?)%ZMl74E9Mx)jFDHaOo{jJrGS=(8S!H;&8KAp_;BRl<&e4Ed%m??ON -zqA)&x(tSU2r{7QGk0HpN6p&=y#!0Ygn2{58}o#UCYgC -zCOfC@@f~zj&aX&EL?`qtcw8krDskyu=;%aONhd%L)bUi8FZ_~=hcmYmc4(?^CUMZS -zxKTf-;|ULvp+gJGVY-Fm5gv9+>~lv9b<&+OZ_y*xD2+*emc4R03>P@K4P}tCd -z{-fg5P~250V0Sdba4i>)eQyF(@Kr6RexT)&VIBj;FwyPDWx0ESiqwl!K@q`X>9OA2 -ze?YB-+636an%-OpRM0U$E(|c}7fa+0b*w5+)mbqw_MuR|afERFQg`4klniwP>K4>( -zs5?;2PUozBlMAi!u041|kH0~@RXMrKo~yw(7(6=;E95B!LZf(r46n0uX0r3wfL -z3r^ElJ -zcBT9s{y(+*Qv5mHSD!zp`|9)ObYK1doKEQ;r1_Wqls0+0lbzc1%T9L6M;B1EeBsd+ -zXtNGG05{!|-Wohn?YPr}|-d*{4%^C_kOcN9lK}udMeN-nPT1EU_{v -z;r=&!?*SLp(fy6j-CbaTm97E{QY4BU#e#~uVlT1x5{V)jTZk3=V(;Bpq7m%9_lm}v -zsIhBovG*E#fqlQ{&Yj)M3Z^_ye((GL{C~5b*>k6zbLPyMGw05oyUP^%jOt}JS -zRXOqb#+H`HR;53wOXYcI#jnukGC;o+`Yg2$uKV*jZHfYHk2pn<2iXRa31pNq0jXJL2i!>NJ)-xsjQH$$?ddQ+?9x-l>>Qj6uCb_zBTz9G;S20vlr|= -z?tmR67s*}nk&IHPlwXRJa!I+Rd{S}gHOk+DjHBeLl;@TTpga)u3P{DISL_e=PA+$r -zJf!S$`vOu4DG2Qe&r7+zh3O<`$r^J{)duO@yYhW{NaknuS(7rJtpX(jDowluh#m -z^4n_qYUXG*X^Ls0HH|bqHIwkd*a^*JxCSO|qV}*ho36fYx$d2=9;?q9u!eZ{*O)b7 -zO<6O1n&Z;~pO*Nv!lyMpU*XdRpC0)1#HSZNz47UTPa-~jSwGevp8@y`#AgsbgKe{d -z3;QpAYaCpB!pb)9J9G8s)ie7xFB|6L;y~m3PXg(R&<)Z6_5Ckt1?db$+DiyR3_uJ+ -z6!}yq8Zkg5os7jp-Z@wgW#2D6CMEtA5=-`u;5SKhRV6N(v>Yx9s7C$59O<` -zZLMtiBF`R=NZa!w>f1v>hoA`dc_K~a;#v(wJ6b2iH9?Y>y6<$YPLsUVX)&`2Qn1HI -zG({PeQ(x6-8cTIr+=~br_V~njQacJ%k8n~u3ikDhKGh{iLDVHkf$$N1L64w1P5o7; -z=~|u6jGn-Osxm!wns`yCiEnk9_)+sqWmHG-n;AXPM^uyRi9S-=zCQIwUEjW(Xj2{H -zgTg=IN6;rabgiaG_(Xpw-=03@sq5R9Q+ste$w$z`Wv2Ml@)UImQn1JWSLCDSpXjUk -zq4t6X<=f*IWdub#A#X~5u76Z6>Iy#X@e7;;DX8gDI7wk^xDoRV&7WR@Vk#olBmUL#5%MYSSCtbE>KD~F6iBZ?@vu@(Wz?R^s7^tYQ55)* -zC6`ehl4Wg_Q55BrPeI`Cf-;Ks<)TmHP)1QrUz8E9l_;Y~?MXHiWCHji`V@Jt#X{{w -zxyVyP6w86|ih7hGbV4pv(v&9x)m5hjO;LUn{M|y-6~tI5*FLzujO&kf`NdUr!c^Cd -zRM(V$2JIdpR<^q)oG%fnKGhNU)wBd(1T`KSiS|?JOcCE0#*LO3B$Q`Eq`i3BU#IP@ -zKTBJ*>Dn~yN9_mgd+j^zTkRX|YwauTOYIBoAKK^IXWFOQC)&r_N7{$l2ip7Ed)m9& -zJKEdYTiTo28`|sIYuewnSG8BPm$jF)7q!1>FKEwe&uPzU&uC9;PiaqRk86)=e}*@F -zK)X-7N4rD2MY~?RQoBeyS36ZZLfcCltF5dxYA(?qO8l2#p&ss^pNgwT~ypxVdQBo4~m1nQ2343GN -zZ1J(QPxrK=qCrmkbRU%BZx%BI30RU?6fnNCqw*1yTS@c5g?heF;}77rjAbY|*V@r> -zMGv2(&$T?aqe42_j(I^kuk5I#L5c&VgOI4O+Je1wNd7C!LX3jWj*1gT>u~yDjGWDW -zh*tj=tcp9?)poqw>)8KG>8$1YZ%OFCfp@#b(bB+fv@_1i|JPDk&)=-&K_mdfIB0WtYTZvsQYa5!O -zM2mbV5!Co7Po1WIQX8sIxYT7x%6S8_^T5PKU~M~ -zrv*57&9kW$VG~?@zWhjyXY=0{W}7Cz-eMf+`9;Wyc&%yX&GZV-!YmayzAKtF4!!lo -zTyJF?=TX2rHG6H}U;S$a?hMKkIx6f&o)Y=Y1@;$`B1#qOP<&F!^-;f;xgGtgyv3xe -zq^*)x^?CJcHGi(Xrq0BAof?=Lc{TaH>C)z1S{7`5@2lk4Ms0O*YuY#M@Ui2ZPNln? -z>e}(!_X(rBXYV<;S7@JwiMjjD>+d^o{2?K?ULR#;@kLy&3*(l>8ZX$`)_EmVAtt6KUrQ}z1cRbQt@D&sZximXc~^b(H?v%p71;Kw#hIF0`hDN&JO6qvk!8H{ -zr(L@I%bpKEKJ5ELv$17&_^(*pW#oWcwGXrkEvSu-HB<~({di*9qhrUebehN4}_~utH)@dery$C&3tVE1YgFbDW -zbbqkvmqkmbj~?FbRo&YK&t^Mtqxer@$v#QC_@33`!WubLzO>~1mRTdW&S-OY(3`+R -z(dSR+`J?1y$F(_{#`o@>y=w6$wSVh+#O+;?`-f-UA3bK%(1Orc|MULT}itQ*};S-jU-cC;YQ_?YmBzS&EO;X#Nk -zG-b2=eqcvypJnm!e$s-3Zqm@iUDA|(X;M=>Okph5vd%K!GQ~2)@|~rPrM9KCC6C3^ -z@-h8x`pNX|@D<0U_e%dNy=r=q^qg=79;BU3OHEsuHY;spTJN+DX)V&~q*YEUn^rt6 -zBCSwbfq{hpiz8hb`7Kc18}(+Pek$5MfMb~xc&Y+lFW_AW{3k)@W4b43mIm!M;9&@O -znGc>)6>BH42C-T;Cf0$?igjU?V*OZyuZywLZEG>h*KurB+i`4n+(uTS!(ld}qfYvx -zi%Hu1-6TnqxKGN_|E2WR0E;wqfK3`?Yp<@*vWqJm*!dMM?DPshHgiQW=Dea7i(MYa -zb}k#o9xdI-8ZSM}N-fn%x}_$m*wRT-`DOd0BFkS&uB$B4_bb)-#5LJ)x(_3Jo(4!Z -z{~aLRw>$?70=y1LMt3hDnY|>y1alt5}D`njoN9J7{g8AktYu -zS|O+{tri5S9zl^W+6nvuuUOASxqxUY2U>L~kLs%-wWBnxc?8w#8}T5bpf8|3UBM64 -zr$D$UPOE_q4Rzs>sLExn{MM0mSz=cce9unmd9_k-KiWG=GT??7e3TXum)3r&`zvTrIn|}K8Xx5edg2;ca$Nz{d{LgjN&XrI -zqW5hmZo;sRF6s*?@&y$2 -zMSGEN4+UR>9_@iq$O_uG7x=S+0>5Y{)*q3!hoZiKqJM%v0XdhpxkdG;jPTpvgNb}< -zV_#;kZ$y_O_v4(#z)_y2subwdWMNga!#F|UI&^>7@ -zCDJwJQJEMQB;{aFU*vxRancSqydHYyqIYFZcqzc3cVf`wI$Y}!4N9I5pf6$oq8U@k -zOq|eR3Vh7SvHu7>1l_T9X~D@|i=&g?;_TvTpbgm!0Xv=GJPP9QV8JX^G7$|4$3H2X -zGBX4me}uDyFPsO4_z^M`GYiNN9sK!;TPDQ}8{gPimn3SIS$UAByTyW&krte#l*{e# -zgR6|~?aHxH>)_}F2C})idw6V7}xQLhuZ -z<8zKH9BHgmngthY%)&3WiL|!K@95~Z+Pj-yub>}tpUoRk*5XTUE@8j&J4I3Yq2CI(NLkso<=@UkkEaOG~hjgy_hy -z@JjJ{8dhQ9V`|2Sr`Md7Z}yi*^G~f8Szt>;R%lSG_@YE7k8>T5<~ipD@5Ez;0&d?bs;qLbuh -zhORMd@hyRj45?gafpr0-Z)G_`3Ta6v<9C#_l2Q&^1Z#wvPM{7f66^>*I<(Xv&q;#n -z%V`IEI}$#Wp;Uso$4m1%D(3+_0Xy*H1N_7TL2Bg$%xDiI;{^Opl#6^Smy~)E()20= -z^_w8I=a7~r1c}M@@#qKnOsR*IT#q201`!p|(cub|6i9d|Eg*V_G>(@k_!R86NxwIo -zu|1e=4I>$wTAH!BCY&{_&RC=Rj8$sMunT}W)*a{M&5SLW#Mqbf@h>`?u~&K&dViab -zHLZmtn9SDdIng<;pIg?Z^WCc9jZbzr7n)Y2nV)mZ!7ZP)7(MXt6aDXHq->G7}5!DwoaNjy8wBqQ1s_}cjbAC7C=9|!22P-@}St{?H+AVH%9X)OAnAg=cO@CmI -ze|p$_`0VQQ`VD+w*wf>c7qcP>(R -z#NjVSjL&uRo42*bUQ93B_{SA>Jr`HfZvUf591K>$>w1-v-jp04_;%x@rU8%JG#YvJ-LpY0@?>lIPMa8WH~ieZCLgwrSXr#!?*Vg9?K{!zizRE~sy*D)yUyJw+xN9|PklDK-|c%F -zzOR03s3F_*89(G3HE7(Hfm7?x&ll6(Z_4cZ^Vfer=Fozb)f4mi2Lx5Fe7xbHy)~UT -zHuSyRd*=6BPI>%Zan#6SsgE8HKDsDR_<=e5X1*G`by<}aenp?<=oM3{-K-T(Bl@oJ -zbgdSi@JF{H-e2c`mUiV%*vsq*xl8w~=rQbE>$0blUWZzi)W2*f(&*MR%gE+m>7}px -zP1)Z2d`f@Qm;GnF_&M>{t{dHN2CZ4_T{>@@CQ_3(E*&Pu`z@MLRpa*5V)7mDpKu}k -z=8tP<9>3ds)B3s<%18K3IJU1z-4&xeCQaRy$MKR=+SWy@JS#^WZN6#j$GJKuP3c&*QvN|-&Ds+EF8W0KwO5xPc#-!6N$SdDwv+hP5FFy9!$XNrg7G3`BmtVi`+$Z(s -z&Trc#4d}P$ixQ9XOc``x+k$81=AFJ8yr}i))ZCWYd$#oNIMt*6*kz0F8FHo{i`{#@ -z?sVs+X?IP}&+X8sjGC6{^}b`lt%JP{QI*P{X)*mluUDgTuIW{_-o9rSTE_)_*im&( -zhkn_b=j~Bfulw~#wVPdgwXB$<_T!4*osZigeb_PQpoYFP7 -zkV4m;kM=J9eZ=|`&ED0UGV563Ik(Ex{l>D)JA233ZLc>Ctyrq<^^5)5>~4Lj(b5_V -z)0(y&nC8|bwaafOcI{nMu5QIL`#z5Bb|@wHhe0v(zwEK&{PFGKukH*P6_NeJ%Y%cR -z9a-YJ@2e!PTzPiZm7SG$mgqk7R*|P2$Il#ps9493H&#c4W;@&JXkR6Q4YtAVOv+eet&p) -z=@q$hx^!=6N$S6Q+uolmp0As?@vYVe$}HOyS+dZ(8hhG?Uou|_E?)a)%bVHewu`*d -z;9c1c*K38YYv$;%{N#x21)inE*W7a_py;}|rB_Z3Fn>F~_^$7MZJZubFWieD|L7FN|=0;PGwnw28BO41HMO^_uqAb`>{G -z`Pihyp|EZjtH*3>`Cfl@#TT{mj;deS_sF)WR`&+B3D`a0aQ%n=2^;e+ewl0Ef&*tC -z%s&6^!`2sb?p*A$u61+wUi~Xh@iGiQ*X~++$0dG4$`^Y5?U2Ngv*z8&-hR-X-ugBN -zN4+mO<94|Qx3sw|=>h4ZuAd8Bo|qoBzD>34344=XxyFq6)_Kad!9jNxZ#PxiS~&b< -z$8K8(H+LMj^oY-|xr^?IeEU<-z%knv+fGJPIDlMw#;QU?af^)wt-E?ul{U5aB=5^}b_MK~R^*vQyHX1ni>#rK; -z+V0@y7(8M8!NMiJE)_L)cA#_F_60ti)fZ{Eqv(nXKG9?S=gdl~zjVWvA|t$eX*_nt -z&ue(ean;3gKhy}@bY}FzPF2Qq2wc#&NS{TQ*Uo&mKTk^7#VK9W7mxa;Ufj>;@;FQG -z9hdAX6WYGaa6bpvUL!X>3c6f(oc}e`!m^ILnkKiB#(z~|>gX2V_UK)8OOqq>YADdX_o^}N -z%Z9U76j|gktE5X$?X34f=Fa-YzYlc&Lf`Y@sUaT+*7G_Yn$P%hbV&2)uxzK!<-KvG -z=O5?8r_KBQ`GcPI2mU-&lW^b7RI>Z$$T0}cHSvlj!Y>}G-XRP{int#U$gnom)emO5CV)mP1o&6%Np7ttzXhU*~`72v&a;ZPL+|Ifa`d8XjEVbOU8Gg}~ -zCbvxQ-(a1k->Hd{UyfY#bio)W&v%=qMl5M`x=YKMMe1!iq+3&Xw+SkW< -zaEsbgZnp{?wRMfrz5MZ#2FD^_RB|o8r1QqU1&6-d7I9(fl65m5&F!#wYDiG5=@(tO -zF%DOz)D7CQe|JjW2ZvX*`K`TUI(D2@antkCJGS@-#eS%fZ(P#5mHRih&r`KjyENU-ukWS2DHS*A -z%=FlVh)rvX?R@*dSR;S6!$o`lR%iRQ+shx`tUtV8i*CDiAG%v{W!oEPPsG=0)8oqY -zwJrBeDe%pmQ9FJuH)3D0a^BugI;Cd&c6GB&3j$--MED$gHqxi!)wqMfwK|S%mRxV} -ztJuB`3zaI~>(p%Dn^7mcA~i!UyxF+--Wr#%7qwp(sh0flx8e`NLfSn&S*X#&MX#M^ -zCzlLt@O!N%h2KtcGKaTb?A4{*4fFWu0c~P`o^)pJ!o4HvJu&HCv^pE`)WPlhnyLE> -zfBYu2bl-J_`>mAv*1wT=^!mb1;~i^!Rpj`tLO=KD(SP3C4_ozTzwc3_>m=`rKaM#x -zYweldD@wQi;%t@ShHhO(of_m&ZCU7(W=CsRThgOI1#NL-;kwf3*S99j`Pq14&VzAd -z*4LP|d(SIrT6jZ;OKI(XxUr#iAM@JW!xlGd`DMbGpOWi*8{azYkf!a@JF^4W!om6m6*yDQwk^FzY%U2`fm -z7+kb$WBuX24r`tEzM56MM18lyo!7*VDDl>5Nr}DPpDzt?pEF{1_^iRdN*?DcAAaYT -zl-#mk7q98RXD<`7W1vpoaK_%VI_IY${a$XWu=#MlE&3AspZwH$@R`Y-<_%une(amM -z!*B17zI11+wxV~9b^+{A$vTTCwlICzP(An!evYhL?lwa_8m)4xaKp9Ck28zqJUDsV -z$0p+nKR2vc;n3Kr(ZJXHYK*6HRi3ch?HCUWN`f1GCc~5Q*e6#LCczx$%=ND=gx0!bMcEi1~ -zBPZ&z7wms>ThGn8${l-A=HBElm*1Y-=}KEEb=vs~txvsbYI^SXe9?kxelsT|n6z&y -z?HN7rkLeX2`&K`{>qf`;7FEuUUUv8HVW;37-IpzN@;(&EN;g?r7k@MKp;5%eb9Jxk -zX6N%C`Q$@E*Y!(UdcE$}uw!-Y`&ABA%09VQZ)Z=JK0W%Md{ob2YwfgKd*IhS`(@(d -z;bE5>g;ezju5==P>b37GE=vqzOIG!NQ@`8lFR$JF@=e(BmjgQg<`CxI|H2o~Exil> -z^8RtB`$?W{<`!-^dCfWFg2~4m)<)GXurI-|FL3g2-?n_|yuVBC$bjP`XAC@2>P4<= -z4-O8G8bAH`x`1K+x@kw9AN1Wjd;d2r0;OJVdo8Q>41Kj>-ON|dOXZ#Mc-zRnd750^ -zSKxeg*S-sT9a@;Z>8tKNhIYT@{YUD?rcNs-&EB@JLxK67eq)0g)t^`EkVESo({oiC -znyu-kcJp^jA?25pFs=T+_JJ$qXT10jwtsP0wKp}2967da{lHNjb}p$K8aSp!=iRNl -zI$zwrtL1^0uBBcE&O4h>MfbMmsD0Ad(4}n;gct3%ZP})=W9QfQ84(fRvf9we?$`SE -zH0vJh%(i4|tq%99N9{Y4dZ5VnjSps9652* -zrg@)=mExU4K6PrtkiV?ghT6o_;HQXs+kqbh|lk -zLKv&k@kQr0k!QQViHbJ5e!n>Aa**-umBgPn7@wV5a^UxGp5*np)_UftUMbyb^zK)# -zVXhffue8Z?J9#ym+a%AaKIKLiYE@)*iE%BG1B$wpD8FI*hNHpP7Uf#q;H55iz0~Rv -zja%oRUg@VtT@DZZrey1{ibre>a<98Ew)6DBv@7Xpo%$rj&-BgiUo8KK631_^T|6K$ -zSGU2fTdnO{%iq~Au4A(Yo63g0FXK};-=J!Rrrz)SIBDRD9|A5e$=^Bn_`{f=E3Mk{ -z`fHEmHjz1+#N8{L<6%%xvaZhW_f6MazBw{v%U4$l>J#_hSdDaKHu -z@`t<*y_*MZ2%Ir`)bg3%bnYD*?9%Ix-TN9ZTr#-iySrn1<||aI;II0Fr7u)2f3s4- -zEoTNqzBnCL`sX^~`~6C9_^w^uJ%KMHFLw$lR%e7y&giK#w=}N%;~mE_%U9?-e0#9w -zg!xmdmr5@k{#5Vdr<*jO`tmc=Z+*Y+ePrYG|$x^dk)N| -zpVK@csLJRenx#&yD^fuT%zl;z -z?GrXOy85_lk(p^#Qh)kpO>FtvrI-Ei)(4)AI%nvebJX#o*&J40yyke;ZU4rL-3QjnbN;u2 -zkw=YNntAtHc7LZsUH3CzHQkoI(x~y929DeCOY?jCZ=VePedV-L$sIQzKY722d%Jb= -z*#p{-9oCqcBBtZ)!c2yp3hc@t5Tr9o9@+UBC7j -zMzoQo3X2eWW@o1VKSv=BlH~x%B0Dk&`Y4e0@hI)3K-&MKH0|*bq5d~a5u{kqlbIH}+0r6kz!RvWjw1j6 -zt1yuU*FL<`^2a=nIV3r$1Eu9W)FYWP)FYT#J(9I_Rx1yobc{o4q7KxKbc*W9cu+4Z -ze3b8rdc>e7>JiM0E_Ey66@I&f;A0Lmt4BH>tJI7AOg)+tnzK$aKK!Aa3^T(=y2f7d -za`rgudNk)WNd_q4oW(IVFseDmVPUn@LwGRfQF{A@DG)5kPGzQNANycw;r&p*K -zmR?~LV#F8e6|Uhun!~uJ7>9RA7+^ERNr)!9@&PEX-(*M7fJF6gJ)l@t98{kJP8`oW;n -z5s~=!2v4tIR`oAg!7unXpVL9bImwS}E3JIA8U!Hnqn95n -z7d0-rG7jH3&T=}Uf9o9X+RQK#3*`q(khYqxo5OVcwc{`vz -zqW%OaOsa!Vm^2|X^CPPLW0)1c5x}U9nem&N3BPaTH>n(8-aJ_jymypCenai4<*MZb -z-Y7a{xodI7?*a5;YuHWZDh1=?D&0h?7jj+MUDWsmHLqHpLLk``44x<<+nA0wn>6x* -z@TnejGV8#_x$t -z`2EB3-a@i+hCcY=&`Ti8iR(9(n;7XMmZO&6A@AbsTegzjV(wBNeB7m5NPUZ34f`E6 -zj-cjEjJz{i1_Fa0FwrlveFWw|EE-zSWq$aK<0%Hs%NR4`*Bt+;eCcVjGfqC;k@~}? -zaBarF+p|gF^gSNG>tMOG?5w3Co(602F3NkPoLEKV&~M}GaQz-NC!xh#3?XT(0l(^T -zM|uqY3ActE6$E?x@MrvzuMpAUQ+{8`Ub&1?{=%P}lImE4KXc@Ys^mCI8Q-S)<&*ZW -zGPM7v@}=2yEfap7aT@%U6ldk{1>`fvLbBq)gWa%1Gy3OB9$p}IL&f7;%NVq*TfI){6__dp<60ffjDPSeeb#}&3MYTWI1#L%eAEgG -zp?x-xIi+X@O?6C0S{>z6AvE6*1j5hpAjqb$`toIY%{G~>-?TI{;jd`ZWSu11azSUv -zKox^=t&aax27lInye`fA4|--*Ovbf3(r*dT=dX@|=s#nJwe>Ma4d}5E8oP?M(`sSI -zZfOciA2AOEebxp2S4A~#y8h?%S8B^JAIuh_@59Ei>RKPbTbif%_JJ;A>=EVrU_g;3 -zNSkOhHqXr94PhrLQ9X5<#!sClcS4=^KstFWZ+{exgYD8LyR_LZ%}~KSfVVG7#8Dw( -z1W!*vTGkz8+luaqkM41nzV;g_g-<5#Eppw3`?2H{tzvcm)O!`tCz7n6(G+3aX?FgutW -zP0o$*D_3!B8fwy+Ruf*aipATd5msm=mMJ|6zj2j0y&RxO^gvCf2Wm3CW;oPG^hg>+ -zFBX)Do`9dHr}+#$KTwqEIfJ63+&c%8Gxe^KOfMLyiC#TWBzhcvo}Tuz^u7Q^E=^FB -zduMXs(rlzilgV^!(o9g3%XIt_<+1C5H-!7)_r+-ZX!HjBYvY#{y?<-`vZnX%jNj+z -z{ddO?lN2T)nw4c`umYE4L{G|+-hY4mh@R#%^#0r9NA$FxrT1@) -zUz9v{k*qLAF;X5s8olU$ZTzyL_iv3~*7W|J@%tRT|7`p)*;s5>z8>R9Fw-EOP^D=d -zHg&U=(^|Y%m8KQwT7s=S80mW7+0rJYQ&nj*(h=Ql}6)TEC6LgSQ-B!<|H{ChP0Ut$v=dZ_W!8l -zdB}Cj`s2A-NF4~V4)8=H4}TeEEW>oUb!dM?!FN>0Dlxr-aR_q=y$U)$xs!}o>Sf&b -zPzE2@2*10MI{yT5Ww# -zSe0kdo+0=hToWLE{@W9{oIO19I)`+VSVhs7%4j=;@xJ6j-CUB+m_YRPrHvA3>R3{s -zpc{pJ4e|;rbi=8h!e1P7F-9<_koln97C#36HDo2^&E$H~9unjH)`P65?j4C$_E7Rj -zeqA)IQa$9qM*eY*$K=R714$ORGYiyc^hR@1l5rYj3t9X7DDW)c@0Jny>oHwOV_e5; -zSQV+tN2OKvPx?HVv#2K6~s6Y9He2DJr%wdg7-5L@x^(chIAt8)mPHP -zkk*68AHkcNhQW@;NA!(wQ~$04zn9XUl3KIS<6gfzTj5~ -zpd&iyyDs#5;3==-^)1TOa=(W((eVPVi@-(KC%~H<{$POS)(zmL`FaiYLX^a-TZ_8=DcE2Erf4gx-+ -z*#&vQO8sQiCt7X}oEEOdd*KSIE~@&v3BuIj4+?IV=-v1rfOaAUC9Hb##yp7kI* -z;KUfYcmh_TIV9%gN|e*QbYjv -ze=z8nK}W~T{&*5m47=6-G}r9sBc*9B0+;^`r93B~152{2Pd^Ui;Lr3rjGZp@sw1ny -zbo{!7BV=p#2q}rD>_MPQ<*=6_g^{1ibl${+JwHQO1^>(7wG7}HK+F|?(H4Av(q=8{ -z5RKKKQDl-w$YS9B2>2tQ$0xk%{K?kX`lj5{EMOd9X+V-yP(6>3?dXe+RR|;= -z)H1NgTM+f}f0ACr$9wRhW8N6^3V7t?jXsmyH4yVXrVG;rkbeAub(YGJhD@^MzXKf9M&QIgamC6eH=~QhVk$E?;L483B6foY -zMa(xVPA&Wh31hBjv>6nV7%Q$N)8b1-L&pVEJcRnaqN?}MNLn;x=HmB!Nnj4{z+ -zks9Cv{QxiK7~sfqcbYYJ0kB0jvKqh&t -z^5DD+`J9vc>J6VK8Gaqf+)Of(@$Lp*tWPo2o@}aGZX{=Q+Jtf;NBnmc=AzE%hPEU} -z7p*M~-vaeWX0HC~_Ilvc1Ba_3<8<^P3~6d#6YcSKw|5=j^8ys(&=YNFtnuQWv74QK -za^3Ql$DDLihyKBaAtqbO0L7(i_6zfwuTpTEvVv -z{s4~aPi#2equmW9ZI45bwtC?dag#rG=t$Qyg#USabD0%0#%Uvs$-3f3y+aA(8C^;A@KY)CZ0c3!N6eIrV!IFh{XVDo`z -zh|$E{jfo1+_LfP^xXi4^BUrc0tOmR`tJR=uVKwNQtcFh(Rzt6{8mJ>%4XkoTAK79# -z1cxI}LOMXP3b+oz9YGa!zx7JL=`Mt<1IY|CmNX3(5Hbu-GQI?Tk_jv|4A>TzBY6kHV;$7Nw_ -z(RQuBy%nbUX~ujo$g5g5e@B)rbR6*UIc&44Bm=U5hfyXhaU3ij?#!*0m}H~2#G=hd -zv=Nq<+UP)2Sl+DKM8I1YmX_K?p^dP#)TaAqEUhcs42!!3aVz3B#2tt`5qJ6Hb#Ijw -z{-3wMA6apuhqi<<7It^3t~_A$e`J53K^DTxyJzR+T~@rjdZth9b1);1aX0McYQPDA -z6Xdx^*TQxaZnL!5XqHA`gI08(qF(TF^&wsN#O~*yb5i{gCFh%#wxsr!-EtygS?ZD7=K}-xlN`! -zFqxSI?{x(eFK -z^wBOxViRRv<53rPwb1{Gkdw|xX{~H4$+OES2a;7*G7@Fv$=SC9PVlV@!QDIVtQX1U -z!oKqQpN)jO1g^8A(vdl>yYB^YtPo6w% -zr))oKXJS7+AXg8_lpxJ957=B`ALpQcamAbUfD9Ibmaws5xK`U#J=*Kho}fLRTg1Bx -z&m*+0siX-H*-RGgDz7aWZRGY0Hqw>tMtYB(tt7|wf8Jh_zd8W^>Ld88!SGif{fGW) -zBdjBt`>V98%PnRXAz77ATtEJU`IBv;aizlk9|^W~HNKcP|4s*=!wbgRt2QJENq# -zZ&5}z`y+6Y&89lSW)m%HPqvcmb_B|(O+lo|Zc|#=ZPIh9TODO)?IBf{4w*^_OT#{x1}+w@eji<#JTn&h51b`Ke1>>eOEx}Zm}bzJBxLza(aNyV -zSS_YIV`nDwbgYt&UZJE%a+qw9~RtK?i^l-vm3Z%$VM}nfJ$_&|?a`H=4Jnc0_0A -zPn`c@N`9gOw}Kw^f95z2-=d$CHY;*isKA#Btd4rbS2MJydchAhCII%*f%6fi;C(xc -z*L#UHmA}>Va@}yAKL_%s{BfX1@Wc+D-xG8wE&98EJI@zsAJikd*n}_vu;Shv9#;6+ -zd7aZavxVoc{7osp$6?R=95&j?@dh)VF1TDHkHgwEJiW67=gSxT61_QVlyRNJ`K!F1 -z)7h%%O`K95yo=`(ztk>Pk!SIh9RG>C9LBEVbi!6}J|YykM5F%_+ErKJ2}M4)6gvCe -zdAnrr4fYeTm%e?`%H0fIe${hfA?%{gZEQ#aOujP;_c=QS$4+RF#;N?D#H70tmE_H*( -z1i+#pADSnrNZJQmhO|}^nPXDMJVGI21xj#l>EvH{~I9(YFBxt -z(jUwdlJ7M@8qc+fy!;z5-Peb)3yAIUlJ8GB7<&RQrezR#(W4A6@FoQ_Ry73WzR(vx -z#!6z}zadtQX6%HM1Y90C?fvwL6DxrK@6_Rq11*cVMZf&k0%fSKgl25Nb3W#UQXl5P -zfR1iZSaGIjp-f`A5#{9?X;++3rFUWQb}gQII56Vd8{ZPzW|MzW@Q#~C{>j3|C~3t^ -z_^(hii%@X;;9rWpm>X(40*?=%3(geLn}gVALmr(z=T~l@TxAFLX=s7{d+?tPSe)=i -z1AXwLo%mz}bO59@z1>2P-mk>7Z*Y4Ij+dpvN%?i%Lw&J -zMkEp;tBjn*fJ!m6AZ?Ho{VIuHosGd8GiBH)lVlWRfj*~yDi&nw-X4dZ(7_aIYvfLEvf|?0@m&QK}N6X`Bud}?D -z)FYcV5suDQ9QOUQNi7;U$+8X8s4d$_nxt~jvVu}UiTaUQe=_!kY>NaN7lYHQ<)LF5 -zoLh|n{*0PI+}q}f7$cIin=DfyWrC#Ng4aoCC87UXHc7$+67jP^Xnzk!Ra1_f<3QVb{L-+ -z{1Y_w%bRnenyXYo`o!lu*CS6m&4@#9i9&n8@9_2*ym~1f9CVQ7*AaMopp3Q`8$;(} -z@Nd&HH!6vZL2e7wrnQ@|Xe0^SYBor#9&4ShEzE2P!Uz$cW*m+4P)aDxA5n-l -z97AE`B5G&_as&6aHHT&)@vBiF|6K^7rPNm!=mt(Zf-cPvs>vb$We8fAI5x8SahSuR -zS1}mJV6>tm8#FtKwglRqN}3?irq-Metz4Ay1O-lm-bEwrDh7*b!|MpV+CQm9wTWu5 -z+@5rbe2Pd}Kgq*#{ls&kHQomt;zR{Jra$wQXl!V;p;agbgoM9Fx>$-_RymN)1v_y2 -zLmEvZn@!d?oP$*Y&n1hLxlct}+}>K7FgA7&TFl$JDQ(FwBg;pS{4q|+*7naR*{>Gl -zmGHIf26*5)D?CAw5+0ryA*yL_H8c7buu32olCX#LT@9 -zlJ^y>bgIIp7?{lh)UaDoNS39V9=4J6 -zi+Clij6_m%>MlEwciCU5lk-pjZ<`^m8z6a#FQ(gP9`kaqLUN9do!l(sbl)J#${~{u>;n;xt=9 -zDyeCC7Lcbf6|Fl&B$dHdeALehE^4W$%QUDd`iYayvaO`iCY2SIPgu|fR?k1H -z{i8@k6?%n9kO8dFtS&zxP6EeDe5y$ljnmuZ-@_1`07U^D5Sg!mRR -zqx&3^$}p7J_z=0nt@TI>v@^tam9pv!?Y~XJc>92};lP?5(z4%;BmbD{Vbv!UgmdD6lL -zg`-#0$_uT$tgXDDSF~$IyB#suQ=l=fh+n4Tp)TxJ#$DR_$l*H>l8lXP3Ac};XMFUi -zXGD$mhj=d|;uVP+<0d4;udN7~Uon_4?|smidzW)o{k46CfM -zX(I7VPaU|X+mDJEEq)IX!>VXRKjI)Ic)&8Ij5;Fb4B!8(CfDG*=rl&uXI@fWlhsJ8 -zcadqt%VGe8?G(GpLZ8LnvhZPI@T|!Gj)6aI<2z6GT1R(JG1#l7e$m}@F5D}{;NFCf -zg-Dy^v^Ox@_J7Gsbd#tLgqa|XIzgJt>bnH$LsjJ2?=8!1Rc-#B7fu{zT}OL23LQ!2TWAfS*c{@b8iBN -zJP4ngGPsQ=AA>9atzLYDGutzc0ne4>XM$u0uDS7Eali7W6}!z3mD~kInk8@D)U!lg -z2Kl%Pn?inQPrFq#Bai=l-DuUj5(LJ0@>W>?+XyFnMC)UYp+& -z5r<@p5@Gi!^hKKcG<}hZ!I)JBq}^cdJMx|2jJv?(+u7_4rzfZozC_F`!cI1j>@Yp8 -zCJB+>NK^z51Zl?6{;#kOVjs8>>NQ3rjeO}Y^j>Hr>HB-DP6#a|@7}iC>xdsoKs7n2 -z@A-o4at-oY6yzbStGFY{YM+t3j;>Z5|7>5{2|fOMJGj*25mu_7w)X)>;Z>2%Q{OXI -zMVe$yR*m-8i8fya_$oqUNo6#Kq#I&vGmd3h)>R%o@~jC1jj8$`O01Y_Oc^mmL+j`X -zIjwtc^1}uYLrTxCd#Go$x=e$bVr0nU7oI-#f%^lSMCOA -zb>KO8{~mUVp34{oy@`27pya_ho(EJ;_m_n(zN -zuI^h_elf@ODIbZLwGN0xh3f=q+vmObMb=)EHLLEewr1)w4O`@Gvu1zCi|3X(yT?D^9f -z5t=jPcZ<36pZoDy^^@+PxYspCo;Qi0DRTG;0yE2O-(B|q@iOt}{eR-0&W2E}7{nrS -zk0?j@Me#EnOR=PEf(^}X{Y=exZ>9exqYT{22_(ywm4=u|3Ym#!^e!l`orr2nfwB0VL*Bo_5d#>*iLCa -zqAfT%h_TMtOFHR6Ek*zTyS)qsW;*3T_bO4@#rzdda>~$Vn;gWG9B`uf?@A^&#)Zxg -z(6}AOs20U+Bqqa=k(cAInLjOW25(G^(-GP7a<86d(P%aY -zPn+ppjwtfdZ1U=bUx32c(dsrDIz=^jou3sSfO36QxnzmO={n-*vX$e}(2inCP2&7@ -z2F{;ZInTP@Xi%yu)93dOTnB%_UI6}-9b+YCbqqTyKda>B$3B@0>U)j+Im8@O-)op~ -z>X-+z?`3k}x1Z!dXM2hFXluFpUWE5Ok1T;O`8!b%g~;8sl@4jlb6&2}VOu%rFg-iv -z-U2;;jI+w-h-_bpJKH_*=k=PevHwi3tLGHGu-DhNIb|{6i+aDj`Z_By@TTCUbgO!?(7{6~4Yx_5{-Ql*e6Kp4G -z=}KQ(eVt)jOJinhkjK~>E1xRsihMC%o#m6lok6FYl2;VJif`|CsBP^v-^}##j+KsD -z?}+oP?A#sMzEKZ2FJw0d#uFF6Ry8e?4*X;e%4eO_BC -zSlef=EyM+l;t-hz@lqI9h#K-(_^6UcV;fmBYSYN_5sSwaAKCaU?VSCXb1TN@Dedf$ -zMI>zGGi0s<#6b)^HJYC`Jy2|oT*k-tua4_czs$z;|N$wjCMAmTZP? -z)YNwv)K186AjVtIa`}ILJoEaayK?w+Jo)U5lh+!t4`CZyn@?85KNst@A4v4g*n3&a -zYFV*1W6lyE>w=Q~%m|hBmp_dvBxOF2=?Oi}UU7Cgh831;3J;6>8p5C8JHED2Eh3R; -z#3Y-Cfm3)Ao#LXiwRo@L6Q0zwbqju6lz68%n)q29nnA~`eIxlEvdUL@Mn|mW_M`45 -zdo6CV*WxC7E&R#b4PdqRj5vp)C!hL9#Uwt@sdsAsC{mVAR(8WmNzoJh${=K`z7LFH -z#U=8OO5n^Eei^{{dF`o+n*<&{)gF6)j5MdDEKXPhl|>*+)d{~bmr;YdSTUrXu$X;+=Bi?CWO_q*a*S5!I!$O{{z(Uu& -z#V*0 -zoK|O~OY*yJ#Vb6ba{=}q(FH4RwMTT{nwyzN#Ai)2^ojaKdlUtrIn-C$&yHa)CGnJ# -z$!~5#40sBd+50RR-(=LV{J6Jv0)}+h5PGUy3D#S~iqPi09LU}kv!>|jq?_tw_}_mg -zg?2h+Ta9TTbX?2=@iqt7`Om$hg8djxX3rWm@EB+ok^c~YcVNVOC*nPoNs@R^WfJ=9 -zfe)4NKm#D^a}1S{;|O1d_~U!*{JoSMNb!?p_MF?E$Ixj3dOE`Q#QE8DvJ`|9=a}$x -zhJ01qlSclgB!AyT%zQenV6#;=>!L^B&ujV9Ai|RI1U%OX8z=M*p)2svGb}nkJ_Muf -zg;NZ+6YP1x2d%Nh4e>-gb1YOsZxK>CKQ~4>bYg|_IX1zeKhk_9575x*9XfTAOa6Aj -zcYWkgscp>pn>$?VGPjAOj{I7RFpp*dDqj3Ymvpk>v8cNo#UW_yPthLzHY=fNyUSa0=-;A?0=Jm~OdI1jBM -zYkDS3Z);Oue61zo*)G)&Q04M>rRi)MJ)NO8C9S2@&tTM7_qV*N6u*Kfmx}(8uJL!Q -z>59hg8p;Xs9+%N5{fuSTxYu1-(-bw-r>nVjBkPs%J@jg@JyEPCYI7?TgDZPaKz;5) -zgV7=y^p+*P-9zss(OwKe>a&~moh#aBUT>AJ@bv6YUGK2=Tdah(_X%z9%8*5j0d`@T -zHS)!24uHAH*ppe!|EJ?fH4Gm(ol5oDS6$k|fMz+xdCkAsA!Z{-y0) -zC)Z~U&a?SrJe^sNV4*uS_P)XRk^||9uoJ`;pZDBe&~syY)*SIX&^Fv4}ywgxCyw_eJFKX^&P($a8>x44{@M*5O}==#3=0>W?cv_i2== -zExi{e;f|8#G3^D#;Cmo?z{_Y~g6=1@avmxv*yvfN8XNcONe0xG*CYHq&2b5D&sKxZ -z^)(0f;!3Y!8IZz1ctTd{-(?^lBu9ww;WGDafObbP!ZAIgA?sz^xh3Tj -zqh{+non9l~z8vd}|37ub72&Y$f|6$N5usn8pi$DwBOykm2rM~&A4Eu2!hXPC9iPF< -z=zKa(Y7WUK{7*)RBw@q%xe=lf7P~U!m64{D!aUTmCn4hbj_?j|hj2-e=fU5RxoaIA -zk_pXM$54F9LG2lmm!=D~Z*8kft+UdQyrG}hkO(Wqf2tweGq%YqD-97n`(OCK68K*U -z{I3N5R|5aHBoL1OMx_t^|F?l1x2$E|(?&TL)a+(C@ZZpjEmzY^rhiONw(L*8kiI#+ -zfu*u#m}P)vG5%M#81A&`zIBcKT$=XNuHnCTPg<_wKd<*$Qt>amr!Bu(Zop6C@7NP1 -zqVi8E>Hh{L5BALBgMTsiWp^wAxDK@>dmR4IBs~~s1Nx;mvlO!wOuv_QC%w95MEcL^ -zJ1u$GM%;zb>DXtM^6;-hrIP4Rc6cq1@sIDnTGm*WS+-g>;=j+gS^~GAw -zqzQQARnHz;UI6nZ%RN9h

~7O7r~+TX{+3i4dr_Sk^?Yj@)%}`N_4$pp+FS -z3!T|@dYi<8a~akdy^}W>t0WIFyXl$9-MBh(KyqC}*^_G`X9O%Py5&;;OFJ&@yySm+ -z?a_**>CRI5Z%h2M2^QEd`B>+lXVRRPP@AqPOpiaVDJH!^2tGG0?(Mt`Ok!( -zIe2pa$;GF~M^rQSKRqL0mo}K~lln-AtZs5c^F&}=6WQKORDt>AQ+I>bgRP0|Z~Q9x -zYqR5NKVzF@M}0?D+MpYs12AgBug1>F-#HxE^RyJs6WUGjoEcN)%bJrbu&o;oJa?O(@j5^y)!N> -z>INF`z^9Y`5<4+oF2y+Py|nA}W}|b;CS%SNw9*_lK1hBB-kz{o;O)qzLzfO3UnHlS -zn}hxvHgD{qOTQRDAYH)_WoSC4u2G+I*XWRP4^f|#9$CSF?F*T@lip2gKha6=DIFf; -z4;ni$(*CEJdB-J3Z2VO>{^x%a=|X3G0=}M~H>sL2anbQ1Cf#8C+_1ADMi<_< -z(wWLS3;uS-+fY@fO{{jNI?8J3BtuP|Z(^-8wNdt^u85(IE7G4N_AQx+!G#nnVZ@})JA -zJ&h|;3Q4do#?>jClcJkH_O{B8U{?VWviRK?lw=WI6FLpYl(5Mz>E$O1u$B)CfegXA#_M2kM4vE5+6 -zRRbcTB^neZh{_TY43G_}(n>AWO-LZ-i>Ro4n<7O)%El1kh7(TXi-toTU%$iBZh -zn=e4y=b!hw-uHSAy?#4$?wPq~=AO?pb7J+r^E)M5+ec`3@Dr;N5lP?t>YYUUiRu*W -z{(kjeSI4V=RiZaeO)B*Lt=dv@Nz$iSW-Gnr)9O>J(~{0sU#~x%t^REFB}v~@$E)8c -z2^-UrOf~VU$#mt$cs0`G+c?z}A(q(()r?7+eA&39D=tgamf7QL-hYSx2dJpP|{d;ScmV{e>S? -z|J1=N%Dsv7c}H?$3-LtEE2%p&+E}&T=?Q7ag72zpU)RanYaBsAKRkK?u -z2p;GVX1_cZH1AdX;&6vW{mK-ftth+Eu5I0|1h*a_2a6uV(icOE2*(&s41e$dH^qt` -zL+Oh#yeGzZVzd#12~v+?>WeX6k3osA8fvd*?g}YG?KSRuv|;wGJjr_0x?3`4yJ88E -zbqU}L*D4*~!=epYq3d2vx^Hxl!W?TyOK9HKMD6A^ayGj)!W_8QUcB|f`ctM>S*9$# -zOj&(pD%Q(1g6fst}+!ARyi@Gl2;&rTju2?;7tl6SI5iuHjDVZjf -zJ94#6%=iBMg;B75&Gp(IT4VdxAE56BZ4cvn#hOC{|NW*60inJXWHvKtMf6nCj2X?JCT&Cz+q5e$7 -zI;!0^M~&O*J@dlZH6HKT7p9!hN}XP@)8qZ-1yMT1%qyFlSN29yo0^N6``|h2~dUP*C_p$-HYkG7K -zXqldJmzwX^TV}AXdoa2yFVMXi-J5sT^yscZcg+CZN550|^X5ujx2EgX(7o>h-LIkh -zwVg+MbRR|c(LUX@Is2${N5P5O&eBB2K&Sc3id*boUTCw$;r8eFqVFZ+*G&)Z7g-`DfL$G+sk -z^7f=z5SgT>iP6)4axw0L!C24`*ZF4+dEemx@P-U)tO_ewI|C8 -zUCq9=b!K*J@=3Z8z72IhK_2jJta}`{(6_11;X70JF5$n>oJ9qT4r(*JPrq2OU~gGd -z9dpvE^~?e_?ol4;UyHwRdQINmvfd{dA|*9)7S!XWduDrIebJ=VeT -zw<+`Wy1$lLGXuH7NwS1**L|Sp{_DCw>ZR$ZOV_INI_mCPYt7u7dbs0*GWq9u)s@7Y`OfVbA|e#xv!X{~qUOJy!oW}z!(TiV)YU+OkW?;B6~()HijzFPOJ -zm+o3y>z&jSFLRrRc>f{evE9CA+x`gNRn+xShu7!Z8@4g-^!@k8 -z)@L0p8MKt)$Gp`4bof$Ft^LUUt8IQ|mGCNxHcv?? -z_O+i{+vG0P0(l+CL+8lTeeWYnK2LTES@rGizSRi&rs?ybbE~2 -zrWDXiPy2?x>}O};Tom#0(8nUT4}Hwbt~683V@I$fat(DeHQhYO+Bus9yd@A?_(d?Cu|Rw_j2knw%DhfnCyKlT$5}W -zqIi|t2yqhfM2h#Zbl-JIWx10F#83Bmk*-48ujMBDdg?RJ(pR6*z0^YPErjPLe2ca( -zeaxc%knn8nrFAv#d$-T`7CkmUx7r;Wm+lJWeKw9>EY3~i-@f*-*{)JzmgzBVddxH9 -zvbB#k$U4jRll2bQ_nGav)b~c}`+e&B1M2%0^=;^>@1StIDc22stEav%_0)G5A+};) -z$6M6)7d^Ed7(3nfR!?pFr8n8F^=F1@(>}8YZe$gY)25~6)=-mKxwoWR)IWZw6+YU2 -z1}l8rXN7-Ix6l5K6(Z^fmCtrr>gRjaa%rz#EA)vq99Z-gdz3qoP -z4k+hWEui*j6`OUk~pOy8+V -z*5v$B^=s;{=fAJt#eMlF_S6pBe4Fm?sQGtWj=Trz>8Z=NmGb=7xX^Oas{fPvp0BCD -zHG11q>i!l@-%+~(_Pd}hu$-O>yY+;TN1twK#c;d!=G<#vY4P3?^p;nq496a*H5^OM -z%!$m*84VNHM^zn66zXze^b(UKH^L>+x>si+B5gc${5JCmE$xf9JUXRMALp@s`ZPUWSzo*jeen+R -zej2HMtkilRZ+LNiqz#qWVAt`vz0O_c?Pz#^{fRwU -zxBhW`*1Z0?^4IHMYqwN2bhOYu&FxJc4ek3oWLq7eojJd#vd9w+*7d_3itaPvGwD9Q -zSm}t+eR25Ay6+-`G!xYLsVaX-QW#%5$`z={Bjjmt2fBnFQ-g0p=RTOkQ)L62i+VKd#_x~ndT|IbH<+=?Ud(Uv0tM7lO -z_x$UA`_qjVp6))?`1FR})7>BW6SRx95^b)xuJM_Q#tu$kd<%Gr-xs5Py~<=4a?cT( -zj^xG0&JF1qyBZH}kf(zK&(01u9@rqCP0adc`!5@5^8~^E6t_*fJ2I>pziSLSe%IJR -zxu0pu)U5IZ$Rx(?$T&UWZyOk8mWwzW3WaLi(|sT9{dhyV7Rw$_hI1BGcCRLWLwvIc -zpKZT0FmKWH^?vM -zU$d`S?}JU8Gr1XY4()c{9Icr}<7T@e)AqQUIZKj#REAF*5WbY~sIWrq2EM3zyZGo1p{oOAudTUcbh19%Mra`YoeV`}R(tS~+y5oXW*36~j -zWXhFI&r}pKqLwnE#w69a$0cob$E$0#pH|8|*YB&Uh*tyJrpj#eMOB@iu&L<^?eGR` -z#+Igd^{3k3Dl=*G!<>aaqkUP~$gN3l>&Y{~x+dAal~#G9z&l(|KWNDi<7fxkn3iyg -zbK2lw)BcKq8eVJaKk3x_fNVYPB;@IK_{3G#;g3!4Q_4(_fO<*qD@3n>P!(C?#KE` -zcf5LquI);+ot}{0yu^NE^R&PZ`3Hoy{u(;d5r{$8D4hS*qu`%sQj8 -z|1>(*8bz4A6H3;isxr06%)*S)K(jZn-+8N?Q)guD$hdOe>Oj$=s{M5VYsM4JTLbx; -zHS`<8zA7*^EcsRQfAvTPt_R1PLjm^A`X2XB`z=)g*SpP@sR+s{2eS0ihj-5bS9f!-uI}dEG-pXOFxGLYt!F15bwKJH)DqTbjX4k)xWWWnm$vlk -zyR@Z0)ksnej28-Qv_|Q(mOL$YwWL+XS4oMS-4gASTH>o(91o@zr|xhpPhFAP>R6Wg -zoZ|sjlW&vmm*>8rAM0z9l{HDezlEPP{Yiv2G0vrQlVdBV$>Dw^d(nd}vt4qvZcE0K7L30jwAamvio(}GF|o_vOgnxK$ktp@97C2^1NJsE>4TD3OfGT -zGA603W%G%DR;K&TwER?e&$c{;`?C}ARn?9`JH{o69a~Ne+L-P$@A$dysym*>eTMMI -z9Wgr|J#jH%LwA(wZv2k*xK)IWNpkMko}h6m8MtxeWm7he87faHF6)z}AWQd6=zAWm -zKTqj<9(7rK)wrbTJ6`Y6ap`3@ZycxVnAIn{3>jwteb1@-^UZzFW6{yO%Lwup3vCcz -zwZA?5K3BLyp7q?jQJ(e4d&^kfU#E8TztOGSQM@ty9=F~8=#KDRwY;U4H%jA}87x)e -zf6>4F5qBlu?znHzbM@o62cERn2LZ)j})(4NZ|08#1>M5+Hi`<&ohf4c9!ac2LAc3e%Q0Y@Bt*nQ>9PZ^*NYVXPj@gYtditFkEWmU8V62PdrlW}RFm -zdlh!VGHFB5Jdq*Zz3pdvbIFI;JESEeO80Nj{pp^q?oxM|d$!A3ZCgEN`Y2!Y>ayHIZ`^av)wSNOc?r)Y -zt(GSf&Rc8TW1jO*(*5&l+}0Vxt7p4po@vj?IrebJ<*R4A!Y8sa&g8hK+}gBBj=k={ -zjr-g_`@-sQeM6xEr=%`UNf&zC^V)k)O7oSVnA<*25ydwvzgB(i_qWS)u+}e@DI#%i8ECrt;)h#O5oaYOeCrK?7?vD?^)Bm?5olVcpvMN-ys2I_|b#&keGz=73@qNoo;id68y$ -z|Nq}5#p8A>3GW@?KeL>qE=3-Nd<^mx-WLlG?E>q&wsiuq8 -z$kb9xuZ9(RoSAx@2Z%G9I7iy$emt+Hn3MGt_NC*j8NnJUS^m5%;bQwUHE(u=OBg$# -zgm?Fp&_W4g29)r@z7oF4=|k+dl<+Y<_Xo+nLN8$qaV+PR@TYp5<-~cKIB&L}ERiK@ -z(@S(0R(`9eM1u#E=!KpVji5x)14{HtUx^MVgZ1*f)p4%Pj_SD;liP3g5=9eh#Caup -zPmi;LI3E({t#*0a6|bH)ja(y3_E}B%Ehd?Fng0y*oZqP3COK+WlqF@|Jh9ou`G7hs -zV)oR%sWL_E3x!i_+=n~DG31;l=eoU#@l|#1(-Z7lnKeVh*T~&(vun>J_SjX^Ni}Bc -z18W|4T)y@6gtVI(CC+Ipq=x;5#(Daa-xTV>8fwpy3XK;}VKW}c4hF=VGF%;b3k -zz9cnZzByxRzBS`!V*OyN9DjLhfwkJYmTjXm2Vfq@QA2 -zHx+5QP43a!9p1@1E4(%C>vo#7>*Rf~uHmZM2)?%RltwUvR(K1!*@&RLh2EJvuPZgt -z(xz#q8M9rSVX=pgXODgcjkZ3Gx9J-1N8|l{8t>FK-gSY-r91B*pi$~e(D(e`TO?P& -zb<{?#dD6c&vBq8NZo=QGxUg7d?T<>+rl{r}rblWYTcz}6_fxgclwO@%=f1-C -zvU_uBOpKIntF5ID^`*7cbQJ4l)6VBhM|pSbe7SUM-pf08mO9?qRqEg!h~MAD@LudD -zO_Td$O}asH>=G03Rl7g7N>{$qbc)+3n=D!Qwk~{2n=Rzc-8;2^EKT9gbcy%9ojLGf -zX@vLG&eia@(&6NCx^(zEKu(VY0DC~MI9RXQQ0W(MK5NQrDr(mtQn)~wt7eLWX(vf%P6};_j|l!UYtyLdL84x -zZlO1mr^#ifC%Eb|%hu>mBPiFDvJvmVZ(<#`IKR8Ub@z_!ZIwh;x$7cFuXkwqjvMN- -zv=8~BU@jK9qweNog7-Rxlqs3-huW@)sl2 -z?uZ%Qq8Fux>vate>e@cG-i)?#zGLXuRZ{m@pRN*g-Rwxu-|L>|xNS&aQo3)x{#&5` -zF3^8(*MAr4zjx@ri}c^mhROJ6^xqM2(x0mT-m8axuFL;h|9wrD4A(=)>c9Wc<#zq| -zQ~h_X9%r5Y>(C`5^xsW-{2%MTPw2lt(SOr@@_&2W$=bu^v|jRpp5DWl#JAi1dEGzD -ztoj(+>P}}zw!h;bXU>d?4%dUeTse_t&e7ln$4|rvi3eJ(B8zzwEi?}|3#X)dYV^BV^MFx -z|I)fT{__q_F*;YS4Oh^E{qDk%Q7M7E(w9E77v?HRtMi(85_}EX?6RoEQD*JKb$V=i -zK~!4reSt+Sje2HX6kk(?zpLt%n@Gp^UwU2^^+{5{v95og{#y9kuJh_AXBUgQ<6G({ -z_x5Z5MVdDT)bCdp)Gt%7-#vQ$&b)MHK>f~8zkBrho$RY$=6Ury(^o(FomhYU?xB7W -z+!epH&Z1f@vVNhm{?_>R7PRR#bdu_S-h$yYSL$iPEqIllYi|ni6Bw^%q$ -z&4BuBJEuN_sn1<{eGZ>jpTpGWF1%K6lC0zo$OO_4=r5`&;Om -ze=%M@{gxWZaT)b5(hOE*tA<-B7XQF8#+Qc!#@OUt1IO6pUA<%MGBrgXW5hnM4^CFY -zV@!U*IIs`OF?O|{7dL>U9yrE$7nWbOe(M;Uub1`QIYhQt)W1lxazOo7T~NP)V{F^5 -zfn#hN^$U-&r~2x5{utZVSHBC#*o!^&+pX7+RT%5^&RT&R)2GL#Nm-^YlckU{TiTF+ -z+tNeyzWdj2vBY<`bU2L|)Zfx({QWH*P2XLt_uWf&pF0*W*)7z?dfy#BNVanHIeqt% -zL9%_L2lU;=wCn_3w?&;yihtj-!}R)Gub1tA+Opv^iwD$a$vO4WsL$PceJanZPbKxa -zTdz-yxy0FZaY_GRxQxj+w4x1v2XL8g@6Yd+u3l -z=8Nl`x4yXBKM7YqA=&XPd+`fa=34i`XJ1>_yD|q{^0eev&jwt-fA;9Qfa~b9zh4)b -zc}Fhan$7qV(s$&O_nqDEt}|s-PkNtpA;GWq{yKT)BGdkzSf4-pF=;=3_Vd29Um^Xc -zp7x)+zgoA?{q?$iE=$$*S{h^2)G(6!`0=_wu0j9SME^RV_gjX`wKVot&h-PoLLrPe -zgmC+^pYLgUPvEi}ZcDrsv!%@=4`ln|V>IH7vdoe4FQ>VNa>FwOU5!|4gN4JrA2 -zr+CYQ4YPUIl44Ati3kt)60~)XlkDrvx(IZ{V8)b9HV_ -z>Q-(jl%ksU!i-bMJfbG|?bQG3P>SAe4`h6<$J#1mA*pGX`9@V0X57>mK07px9r%6X -zIp<@FXqE4P_qR`R%=TT>XmgwycJYQ-2YcKbgY-p5**&Jlq17r_u -z=wJK#O8@OAeAJ--jPDj@&mQ?E`_N!H>v39m&%KG(8N1zu8D)*~v~e!q9Byxp= -zG4>Fnu5nH~;jPjVPwAG}61GHM&>h?>v|bxDYzW7@MY-mhp455I5&ry!pnL93FYQ&d -zmp4?qr^GGs9cnbuQlq@D?6qoEdSCiQf9D(E(2eY5x7y|NVwGyd8r&v)Xh}n!own0XsTCjDqnpBCX7;vIxRv6iG@`%tQY+GZPGkdHRLW9iyaBD0 -zZ~u>`-Z6guw~P9hsQP*8F83UzR9e2Z%RcLZQawPa=A2im2P(qlT5w*u9-s_M&MTLc -zJ#ay}N}A*;YpkQ`##>VROS0P?K6#Id>#>zHY^x_PC{-J!sy?q&Z9S!GKCe`5l;Gv_ -zN+o4&=aq^%Bf`tK%K7$RQ@U8kuDrmF)@kALh0mJz46xL@7nCVF@R9xR=anfs5Vq78 -z=anfsknU5O&uu{|i$*rE1^dgC)-21kJFn@yGVOI|-jqH){RjPT@elD__=Elvmhf5A -zTF#pK&zI)5ciFR=>*ZIplzN(8k{f%iqwdSM|E1ZY=3B#K_lCq95|@s% -zsJB{!mDMp9Yw~_P-E~Lv6M;xISA1bl&u{iVu|I3xS^I0L@)S(|4dv?{GF6Scgpe6f -zHnsQWeRcDKz(`+#ei~HHcpLZ~KWjSQzt`o}vj|+@9ha1>OR!2P%8>eQ|$V;HLahT3~8T-T}1D_Eo#z4k+5&ff?Rc_rDu3 -zsn-jn-Xk(BUk~G6zGUh&@2{Kx6!`7_tXqG_ef+t(ydAKp?~;dJOYX6F=g#V~pVIBC -zRDNXtRJWc|8SV>bd!EvL-}L#E7TFi_?IkT1RgDSTSGMb}ymQ-iPhNk!R>!1saGLxITgC6a%QKJ0RWt -zm1cHe_h`|>S~4!zqWP=j=yxvH#^H|RyBz(mrdiqLmpmismH*YU)P4iKnOkbkhI^%d -zZutXk@6>Ou1MYjZ=yCrW@2~$CUDf|zsp@lmH?9Rjp8v~z-(<&`ag!XMjr*=u{Qmav -z?0)WFxOCZFK@W5LGizEQS;XChE80TYPGOe8&dogAE(&EIi4vYI#Fft} -z=#znG^TX~InZ~tJct%2<;*og@p%SBORVb7!j>ibkvWtYrOS&|`{ -zc|PtTbnKLRoI=QQ2+BTMM505ql@bTnZzGH~)O5l>eUxapDEAc7j9Q5qQ?i{jGP*h>=rftc;^;Y`Q_k_m>0-2Ie6}QV47dG1mW-$cBFNX^S -z{yg>FkonOlaqpoKq6pg1^$6~x@DSJ{Mu;IW6lhctd&nYe5IQtm_zw=JWKqa1Vixuj -z2Q4BF-J$of^#_W`KEyzR8IS`aW|)|Un<^s2Wgy^jZ2bg?2=5kJA7Zz8Umj8PRa -z2ufvmL3gN}cBDmw7`j}1og4}srS4S;-Pt)>vFIZEnYgo{UT8uYBE&9ic7`$V6l1!a -z@uwv+roJz2@{kC1VUNmT;>ai6*}>3ogztr%;14Yq4ycE*Ai8Ba9l}3ch*j^=xA2EP -zBdtcO#Vr>$$`g7&6iShCA#`~tWW)7$FBfrQ85E;qFm*8t6}g71;l{ycaF|KgDMT86 -z7i58%HYmkiOFuW`CP3(PD0EV9XN@UnxHt%}!XbDARLZ46N|&4`=>MesGLrd;I^T!8 -z1nz+%a3MSOVR!Zk=!8Yc7Q@V;v0@?!7j+wkA4CkjUCO}xc&xA?bHx>liXpM0T=Jt> -zQ2{B`W3ZD>`$VjWBTkx3`!#x^hKX6x%SDcLx$xk6aL39zqbot`MVC5cn9#tF9#7RU -zkpq5o$D=P6nkbtUw~cY#31TDhWnQ%P5-5UYPz(=4Xk#ce7CXcc$J{zhlseJpUmX(acX?(E!8 -zyR*M$q#g}*XCJW)6K$JgMQE@LgHP!y$VD`ry(L!m#O${J3WeHMg+k^hh3Cj-y0JM* -z_y;-r_@VE4piM%YLy$xG5nMm6hrG>k%rT&X24|^m -zDPd#ri!LF=len~36|zmTtRr~tj-m~*1u`??W`%NrNb1gZf(LBi0zVW%0pvmz*zCAq -zbFgkox%3Z5#vbJG4y -zp(2iYB!B~)kVBeM()dC6nOhw6&B|n9W}Onut3~v`LyTun6*5pk0~^Fa0yw}4DKHkq -zv1Bomv|aR)v+y;T>1eM>J;)CSlf~2c|NfltxS}YRHcVuK8C1}~1vW^7I7k2oq`+91 -z2u=txR)dV)CJFvp3I2NAdfZmrR@|Ks9Erc0IbtGtxNuvEi|i1xlf>(SGth?Ytn~jV -zR(y?n6897wBVHDH=E5x41^Yk@#wE>B!cTx00Z%iZkj6v$EU{877OSv5^Ot{>n8o~) -z12aKjt_0-Ae`FE)#PJVZDO8V~w;vVCCLu1bVk!fHUqsQ5y|_V;vRv2<`9fq}#{32e -zAixGG>T+@I?JQ@^c}_X+nV9pD=cZUhHfvCvUxsmz_o -zXF>wH9FWOcc1T|fipAMj)4Cw&_>JOqc~2(;DU -zheL3b=VNdjPQX!Q$KW`aS>rvhi~8wu3nbtlg+Ng4fPA~I(ZV;V{mEW!^O -zs9**mp}pjqCq}GfgoRO}K}EKX5Qx#v0xBan9y#fv>lS^Eoc%W>my -z6F`V;%EOw+K0JkTn&}g>nCG+Te=^7>%TD}dv|%wk3_ijZV!KS-eEAFoun@eE3lqq< -zO0HA44iht(_i~uOMG?y^HnBxo#1do)^f@7Mk%s5RFiXaTi82mp9E3Z;1v7chhgnbr -z9#}|R8}pP8w}5AXyR(ckNq+)%0~OY>XA3|XIOX1lHZ>Q-3axIlYcq1?-8N`_t&_aa5uwD+FB&grxIhu7S{PQ -z_?O{2c@EeyNi*9?;eli*2Mk2EPNr -z7|Jc7t{~PtB*vCLBoe>_IUvd&5;kx^3eRR-7fh6{44*)H_GO1XON4XU65#+Z&-sLV -zAs1#s6>-ZUO{Re=@~VJxI7Qwk;REP|6ChSUB#z-8ft}C_^-v2z*aDlN3M#-4v%mv3 -z&_KXZ^c;pZ*a=M#gw3!Cs-ObOp%nb^Ff4;2SOSY7lq|$T+yd}HK6oJ)X2DFz0S}~r -z6C98LaiD<;<$m%3KlmUQX2DFz0S`FA0SOQXHqbyo8uP+L`bX|o)-E|OvY*ajZ|%8` -zwIG`HBZhSWSKuCM#XZa1A?Hcvn3?QDYgq$r%wwgjeNO4J4yNJST&%me4p7m#n7yu< -zbUDmH0(~Bt&n(uibr=FXgV61xZ?QJEfeG1AC}7`{b1h{cd?qN!hJcy4DwGnp914&X -zfrq#r{^2Fbm=!4O}o0G9e3`LH1JQy9tiTwYrKqlQPvq1LOoPq6N2&y`BfZ -z2iG>5xuAu%<^9+!_k^xw@jK!ihfMm;L^#g;*a=-=vk_n2ak;qTSHtBYkpKC=#)yM| -zA1oHH8#IwM=LGrgcbmna|DXy5(qf{(Zz9bfS(4Eegj%SFCV2Bp$&vSCNKzF7QAubW(;c_%vVfY%wEepJ=XUo#c#U -zG90(96rJz^oP-mw35doy<4Umwg7Wz)Ec7wuyPh+sU+~_>x+rKH;ZGL9`Q$@hN3iKk -z);lqj_j267k>6PAs>!&#EADV9;;?LQ<_Hzn%${x{?-3r^hub)RV4sltIQ@lZXEpu! -z6^>?BvtFSuzC2mPA0~fiu{i!AVZUGrBA&`za}-;<(1O19da|7j!K~ml#i-ZS*-@ -zr;E0jKzYbtx?PY!d?p1^gzJYY2tq3y1rMlB&NjdXF3RG8(#w{MZVv7|>}zviCitNg -z%E1Yt52@1x_EZmI%RNGz=w@l9o?c<(QtkH2(IIndr7rU5K{80Nt@~18n6^n?% -zwoz@=`LB%MQfvZgP>Wr+K??8q$AavoDdPvQkp5TzrH~7)lv6CA9<< -zw54od5fVjm9uZw0=S1oSHW1?dP{-E^!ZY`N%fu!0C&+?K5bu5-nS$$tG#D#oVz@ZM -zxza?=m;C4}l{(*iYO4PZ#tHcs-A)-6k_HlZ?pjP<;97*e@1*bGil5L{-7Cal%6t?$ -zVeAh-7KbmcHzztNs%eQBJGVP~i&Dtk!d?;DXt2>ZQz77>`*ij8SATgB6BWOY*Rd*>8wSKipO>f@AgfK=OkyU -znazr)fafK+nYc5Nm*aNQZXO7=36Y7dTd~FS^eO+3=tGiaP)MA6h+9NEE~Q;~Zsj=_ -zzw18w6lr!s8|;ILlw+ccGd0{4T<5)%k+ebL`tPT$c`oAFhQAB8G$@`{$|4>>28Yn` -z^L;whp3l!71f@`T7?y?g13 -zwW5mW$DtGUV5bxyD -zw6(K4TW%?ivS0i#*5tVioi7oO{>MqKygB*?r#tiqWG7(fA+wmZC&IJv#RyLd<#EDT -zFiAtTtqpy>O7L%O$j|;{XOf93MT-qE?2saczy!Y{GGHQ1g8K-|fX(P$NZwrAh_=5o -zKcLfN>dyY;K=Yqe=Dzy36i+el_`aVho**uk%(gR@F2Ri?O)l|gfp=(#z63vw@FYOd -z7?UR#Sp^(}g&SF`As=SJV#om>=; -zU7IXi$i||pNUndh{bBqO;xL2dFz2+KQ?O@nc#1^=WSUp8p05&}#K|WtLIjBy6pCQ( -zhr!|z?k*nTuE(Q912>a|NnMYKDBuJk!USgVC9T9dqNhKjWUt~*MCMDGCcz>6o=%JS -z8a%^!*Kly22&dpAIFKbk=t(-2N>gd}7$MA2) -z4MN&`*bKJ=UsApR!gRWb~^;0&CFui+$!ThT9F#)ekE -zN?6&qXdu=ni!|D5EVQxLRgfuiKjT~_T;yZn*}`*{LcMth5OVKmTE&Gn_XyO*)249t -zOCJIaA}EgnW{8A1`HUX3d6noQe%m*zL%Gwj!6969e;)*U#iQ>r7vT33)(T$yMKbIo -z`jF{SF$^6KeyND%a369L{bVSNf(eiTqhT0CfEz}F0z<$BPOyQ1IP^uLrxhD6z#Yo- -zBHW8`7vlcMqwJG#ClfacE`fNs0xWPTTnsAZJBsZzSWDR+B+Z?$4gR|%S*%?{x>D*- -z+&D;qiSXaZt|?0vKY?xd6Un!jH1!gm&yu#Bvze3V4|Kn>nl90-K-}c0v=>LOleb06k{xGX%F3*AJUv3v7aNr~o(NE|?5g -zz$D0k2`~yq!{$#|ZvvbVKoR(%3S8iUT*yI}2lBxOUKmMQCtL!T!WdXh`Iq84v4Lj> -z>vay}iG7BP_%X=uhrtjH!c;6~%J-raA%?kF0$SMbi&e}ct9Zx4mFKmff(AB-gTu_- -zCnX8MyFiFLwUepxRcn>Vlxd%atH85L_;}9oGq#xD{8oz?#Qmv1Tkd?A%@{jHa{NFL -za-jf@5$`yNdJAVd%oF%Kp&Uw~0;*sWY=$imgj%SFCTNA7unUgD5!eT9a0m`d|6uB? -zkxx2f9aQ`dD2B0+2KnFvrIY;-t`lq!2O5|`fX76euwPqQ`@D5|2i~IK%BgOlF`NJr6W+)WHI;{cLk2rWq -zg!mI}@%j&+2{{^tu;b55+Yu^d4^|EpPz8SWcWrWy7qv<#2kCc^3q`P*@~bJ_v4aLS -zNP~P3+_`vhb72-%WbHf!F+AH~D8#{8 -zWbwEOpkL^UuS2=l5C?+vSwD1L&HfO&@V7w`ejA9dvAOvyJxGL}j_25(G;NSOlm0{g -zRWL4A5v{ntr5m<|gt#t}cMxbIP8HbbXZ7=CKgc1 -z@Kjx%J1DzLh_}t8mn$MWK{kfHIw|_L} -b4F2C~4_;jUw%c04m4Mu7{r~pwLDT;MxmF=) - -literal 0 -HcmV?d00001 - -diff --git a/target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld b/target/linux/ipq807x/base-files/lib/firmware/AQR-G4_v5.5.6-AQR_CIG_WIFI_ID44858_VER1745.cld -new file mode 100644 -index 0000000000000000000000000000000000000000..3ee6ed003935f3ff65b7d74a6fda580a41172af7 -GIT binary patch -literal 393218 -zcmeFZdt6i3wm-VEvPeiW)dVo|)I}hm2pS$D_-F&AC_cK8mRN1wg=kPwTWj5|TG}Sj -zQdGKk2yF$lJxOZHqtYg^tq){3q!!d-3u$+KKrloi4=D{)>znl(D`@vVzkAQ`bM86! -zcYpT}{Pt#Q1jZ2t#Q1jZ2t#Q1jZ2t#Q1jZ29*a{|AS3b0jx8*qr}D{j*tEJBNQMi|o&m+{uzEv$8c=dAh8vrvD^6XQ48O?B{H4 -z=Sc2wr0;OD4{-9TIa`nYC)qjgZQqea{*onmkR_F7WoKvQRnTD|_>c4>7!z$#D -z`8xc+ULE^yxBrj-Ui<&8&O^WXbN?Hir>mx6nWO&={Qa7AGtyF5#LxPb>|;t2bSccN -zd>!+@$hXeKNh@;(lO%60O`O4upZF`ZF8$AUiUn;UEDis4F(F+}6#XWZluNGt -zhV}Sk1OAxaAJh3uNG$x`5ZUWb=40V^hgRGj%D*>+m~Y_O1^i;)|Cv?k+H8!(7J57+ -zmYrz*>S(~$?P*)3N$dUHkOL{dAN>XLSM+-+K!)@xoeG71Kk9*&;;#&ebhv0O=BH!u -zv;48e{(O?=@5+$W?oWSz^iS&=lK$ksqH8G2C(SV={fGYw$56Tgd-A^qI>IoIKQ`=- -zjre0j{(|p@;=db`T%%hJKL}Ka*z770k?-nM*UhHajNq|J~es -z8TbAdl>fiT&953N6}GXk{L-P9-yO<}wvys-p_;*AafF;GPBJHxBjXfsUX?eKA{!-> -zkG}R5o5PYf6R#!Tir+tkuf@VkhB)|S9DDx`_P%{S%8+IV@-oOQW+;(`w~?4xNWvx=5HOUNI#6hN9=C -zCs81YPa{7jg-d)7J+1pqYB+Zj3zm|JDomtDfw~fFaT2B*p_4b$G06L%3d0ig2+y1a -zJd%unav4(p6eBBQV_!FC9Aekj^uDM6kku -zjx@Ur3T6glTOaq~Co+J#9Bci_^R}S1ByIkXG?C2wGls4ImM48%AT1$Fr!Vx0BeuTf -z1MFjIQpsFZu?)jx>QT>bfo%Hd%d>{I&LNvujA57`JCyy3Wc?GLuYa;rvd5VHo>92N -zBfNB@sze_(l$S{A68U+#q%N1gQ$Sj8O51KqoHw&QH`@$N(o;={bF%K(Nk&K|Un3u!sX43Yzv)VaO^9{Z#&gwDo}cZ(xXie6HDiz5K{r^+14cedlTaBYIv;y -zH=JW+7d35dP`=i{2&rp(rO88LbOuCBb*id5tU*uLrptL^fqtE++5+^mKW@^Al-^+5 -zXLOy1>7Jua*>C;}?LTOUjzP6L&tpwH1a*m$RcYu8*^^jEhWZSJ1<>Q0CsJVr`t*+} -zX^GX7Ay{;L=Fi1gi0F{bQy?h4W-VdtO9X)VuQ9eJl3^Y)Jk28#6yKpj5;JeZ`tU&o -z7P7~h0t3~2lTs@4K4f9O(%;YUFu9(LP)Q_ZECT!eVklf&AsOAg2KZNIF4B1)2B%zOI%>WK7!t3!yLVN&A -z5tPJB6V+siI*z+9ao&8Ok=%qe`kZIh6CV;;;j7|H;+NwelutmcSo{I;3;NlYq0!|P -z-7^(POkYau-!U|eyB%ENL3HfYWP~H8Fgd*FL0h6EqiPBrFlia=Y1my=_JqZZvK|S -zs!zSFxhpNL#e$^RWxaGzh27A%>23h0ZKj*1r(CATY;jCahDvfy{R`3Z#Ux(D<(>dR4yfxsttbotv03)s=8h;!!?PLDdXZ;;EZradmKL}1FW#SzL}QaYeM@6!uKBfUj? -z>7pViI%uU4O(1E)e|DLkg3{7+YEQ1}7oV0d1)|ebn_KzMKA!&1_HnF-vah;8u3F~f -z)k!|ckM$~rA-s4yr5m*BOgDA98_nG`0wAA_#1=dj3E$?fJnBOWQiK=_fOfmV41`-(z4@w;i!+~F{J?tg7Cx&pcPOzFf{eJd)*_^>GP$NKcGdY -zn8bJ@nmY_zbo33)jfb+5p&wfxfH5iPbdZ(bY`;wMoKu?;U6A8owj7X1sXOGovFZ$&2w1|Y=FGL()7zD;8H@% -z%>@vKUzdElXaV~I2*YV$N|Jq}?U2>9*DE)vHU|^rMsgSA<`xwF(OdKuUDW=~sAqu{ -z`$(X1c;)74+XJ8MN4@<8dduhClwMUX8k-tD%8_sXo!CHs|2Z{8WBm>3Z3x637l>L% -z>DfIY%rkcmek_5H{OHauoPf=K)OW9NV3;Y<3jH*hZnCaW5;Jqv%kv{hvwujUgbW}P -zttm_?6DFAy^sYpbBxU-iy*#*v;n$QWeL|-rK;*D6?Eaqn>o6?v$@@fL?%?S_#o(U6 -zzYJ!Cu?C4ScIW9Z-_AW@oX(6eZYL4OYdsx?TK9xaXw3+l*h+*=;!6Tbp8#t`BCv|S -z%}dwiWyf0=9TiG*tqJ1Lba7bflUcb5;;HH4X{m~=zesExshuNf%96HcNn9N12uISL -zB^}P1{$xt-V#Tw6!6aVWOojvxOFY& -zWco9%UvD3{-UfY`paDEry&UfTpD4Qi9Ys$*)_vi+$HlbvFfUHEQhaFQQoL{)>umT8 -zmFRQIN=b(=UdNEs<0F7HziMKlVP)PPGW4g-rBEuGrxApk9t;HBk*W16oZe{vE;GlIAX2PlGSPyPXX`ay=`5 -z)yt$;1k!aFKnHMn)?#BlYp=ZyT>~bZOs4P(vp?Kp&GWSKtylfq-@hI@YP~PDj#@pQ -zypbO!xBXzch+MG@{xAvscv!viVRhLH_xr*b2d=l5T?ecm@jd-Xddlz767iUzp^&3)}OdrUh+5^bh?Wf~*A>9l2L4ug%QS@L*1 -zUBz4HpLu0;9@uK$I;Rp4JCoi;0$t%YoP5iN49}B}Na@ufN1r`E%_L^z91a5|V9jN$ -zPJCo3j%jPCLrQN-vTtU~n?C1ioolBdxNlkIAs>CsV(y%p5 -ziQFzp82Lbseeewy#|ra&;(wbLHE&LsCnPF6(dsRSmHxEk6*&_NNtArjjJ9l^Yk4a^<>8+|rJ?3}zKaicil8~)BYIp6Z%NxU|99goi2pSaOy=j8pl -z(LKIXdM2}bM8MSs=2A#qnmeV3VD|{ZJaT_*W}-Y7o0-e*fnmUa5nv$9BRouD5mM#f -zCxcSoz&uiItrke?lso)OBJ$R&Hu_W~Zr!2UIeEhq%Dt++bkmv-R9XcSny*g0G!9GSw$_a;i9$hHU+u;koO4=mY?39W#~2XC2`s$zF#;J6>6%*`#U18WZN3 -zJ~_n)CL1mJw0<)Cr{sDU4SKu=#$>ZqF7eQ6AFP=SabN8tD2oFHBP&~_RY-Z)T?8($;iPtm_`&vGjQ8=0z;1LwK5zKAO}&Z@`7 -zN?0Sr?-B4}U|=g{V+P$i<6~1K_#cR6PQjXSjD=&F3=GsYOgdn4wZf8wmbm{k -zHOvEST}W2Ez%isR7Qe_@$TfcGx15`!d&72ax2!0d_oFN>+W3ovHz4~g%DpXMH!pjkn#EW)_L_z_pMOH -z$2?)(RD&K{@SS*`K=xC}hur3uFcgeRfjCj#h{i|bDR?$^(#g$-1UWnMnG3JH< -zHZ^f{qEVR)In{BplfL3|KikKo^_*3Bgbf)l2RDad`ho#-!9e@^0fgGxZW#X*kE#$) -zypn}hTXdAMCQ1ApXTQZ}d&XUh#c`q23%RX+?)}N3g>H;_b=BufvELpACl3?%;Ggo_;#kN@A|a -zH=?FafT27TCme`0J|8077YWJg$TxT;vc((-+J3?fk#E{s??c#vn92qL1ndY0)Ado9 -z@kiK(F#Auq6|7pQIDx5br3Z*ggSs{e&+aYHOv7+b1%W4!ZcrMBL*b -zAB7ozgk2Lcr-M}72-8m;upGt|e~O2~3G3cKLBd+0LYz<QL_ -z$Bii=VIJN9;^;mq9~KNi4Z{mM@Aoi#wJ#XF-%nuFv3TA-;l+yu{w!38cMAow|`us+%FF^Og(gu>8N<5x+>Gf8Q^ -z{pfBZVSF%${t+{hVg41)X_fQJg<1YWe}6@5a6vG~E040|Odm{`E-=_`3kUcX+i$}C -zd@pOyB$wtui4bGfG&I>1-S&Rct{HIXCSb)Jj5VFgjHZgCsAe(65>uIxRB;5= -zETUj8W`2|g@GC%$Fo?|m6i+h9B -zk#D^dgDjECWx3bqk;$s7gKs&KRq6^aw45t`bIwAp -z`;9pdd?cnD7t?gsn|&)5RAn*5cACaciWYrEMso8h${5JuRmoNj>P!30rTs8n!s3Oy -zuIo$s&3pS{xG$S5|(qJp(*))?l;azke;))e8pJE -zmi-rymsau+GQ=f4;j?QV`4)uL>v#CQy)f -z08V9j1z$`{dL7~!Bd66f6_t@$p6g57idWZyhMCt>VG(=vaD6cC4S -zs5Q2xrD0l;v}Oj{J-3kU-W=eKG{y9{6izHjHP7j{mPkx>{Ms3NQ`v>Swji$cd|E-# -z#6<(V_SyYKcdFlI#)0JZ;YIzDA+jHDzURd%jUoPCsII1Ty2FP2kawyG-bqV-h{BH( -zAJM-eqQ7&Oc*}$>Ts44qd#?CNKiTT15t8eu=A{hxh@|zTKD0lShn6WGNa&7oQ7q!T -z9M~`1wNRUe*4loc81+~sc1TCQfC{PV{8snV-;x)+O -zl&%VX&z1$wMW~eb389TvR<1DQ2d*!`S6-WVs~_*(=!@pBJ{^)_S%yIodxY+iN}TF< -zWVMSyI(ySPC+hF^frz6c;v16Id1DhGsB|a#)9O7T86%4^<6OTZF%_zJ`UQGVpV`xg -zP8o;)UBT6>zX-`NEX8c$fvu$%RV`Zx;~c+#rS#(J8zC94Khdq5yZbCN%TDMGf&g9} -z#I@07?2W#fvZPOfa7~?kQ4E837!kURrJ%=$;Z?J -zF5)QvmR6dw@)`pLwYT(Xtb|20hinzJHSl2-u*m1stU#RJLc&`><0RU@?c)y#+B*24 -z%I)$wh9oIUcr-z502QQ}L -z-{{gGViuhTM17gw+~**S9TJC+{=XJjR!#?x2KFOPeB#&roJ8aJn0>>|Kmq(E!@?_w#`(U4B3*mI| -zCBG7`{NU^%I=6doA7nwIwAULge;Acqqoa`Jz91<&F(OLVDG6tZKtu7BUV4gC{#xMV -zbyFq3wXmL}l_!IimFS7KI)Sd1G(dy()aH8<8)0n~m`>6d>i^Pb`}}Kb1K;-f75Fyo -z?DJyFx5$%Gt9f;wqFu6v8~L;U3|0L3#z>e>j4e2#urU$#S_`;6aKN7_H7+UocK -zZ1G!^KDy658Dv8IcZnf?3i8QGOIgDO23-+4S3VyCBhku{d_l{LgaZHPv}sL6L?#Zi -zUE7p`HmI{OF(r^Ur5uUTPwF#If=MVoAwi**aK7$Hx#>6osFvq9d&32u-0C9ow`=ofAGqnQOfkfHy}{GM}^h-d-?3LQ$BaTB3LK!o)($+@DIo -zc<2^nhEnvPAP%f@RkT)zVDkYA1tDJX`&4oo)1WI5H~-sS#1`qO73X?o -zCkdm$x1M1%Qp;(O6QXor>@@jGJ|m2WeH+=3WD{%P+uqRY`i!#2X`HS6wsryxzcGW* -zrqGawt>}{O#O~&mFyq~Ec)~iVcz688XZp6I5ZdYaogNFLJ-0ATjFtfRIf3z@uR2ny -z|DyMHO2u@YS3)bN-$^lA0`;ceMgk%GPmHAw@fU&m&mKkX=lSBJf%;D&UUVO__dCQ; -zy5>>zJBPv?s5kaDGQGL>^MMLepdm6-+=uJ+j|$n(2e>~A)PMBPc&NKR5bZNp^`0lx -zk&CpEPgXMb`WuI1%?El7>z14+&@Q(p*8CnsmO`XT?1?q3%QA#NulPI=EiH(V0xpIl -z(^T7gwUGkR^fHYm!&E8IiTD~3J)_Wv=ue^zWqky)d4~ATxZUnDUw5sqep#=ZIn8)l -z&`e-w2()+Z=2%0dv?fOPjgK&vhx}keW9-D;gRzLKFbZHhaoBemw-FaV9@dD*@!&Z0 -zsw%(Ij|GQys}O1Nesnx@a}?S7)S?YlJ9lf7-1fsrYor9rej -z4uLB&5=6_D|7`n$UaKVyR7P7XWC=igOz)+EHq+6omO!7<3;Io0!q=2o5m)*7Zuk4X -zz^OWX)o@I&@{4B-vvr32KU-GRXv3d@bnQ!er7}sqqRthb3sKig29P5A`BV1%=b%sG -zmwlVUdM)997RC_j@MjI}4COzPl$s3fl4tEpR=L7ubd}8?!?!G?cO=%N+#9*|(Ru-i -zBb!a3Z}c{@N5Y>&Q(b?06Ot^Ob?{Y-mFoP}5U@w->2HDF6`9O_k$r3nf#w%uusBlD -z2a{2V^Hbv35YaM38!pq@J|MLDE7$)*to((6CV{~+ABA`hmJ9Dh2+xT|!dD7k5~1gc -z-v(nS`d}K}ROB3Otj_dTLU=J$^8+C)3I$a#leHZ~2!YjExK+1Gu*Cw&#H+ohg9h5AB||E=oGtHgC;7a1bf3|>s_LH(D&G8pX!bHv3{cT -z@#>8XG}ZKg?`$Jr>xd>E+D;eQ%saN(SF8xB-@Jx)S|-qD5rQ5>}QM$-y=(o6F>8C*mAx=Q>&(_*?`(|kZltDtq@zQ -zEWS=55sOJ7OEUwg5=h=mjq3V*gnw>D+gJ}qZ*5}tZUlW;ydO3^V`E9)v&ueqLv(b7!lWdf^zl&%(;0>Uz7 -z6Z;qteh!>B>`m-y2reCyM#N1SW_+RBPF$p*#-Rvb5PeOfat -zZ#ZIo^>nCvk3d-GtB<;7j=Bab^y89e8PEE>!Oeb_pv$n4&sS?FxpxbMmsu5z3XbLw -zkqOBh(nDxWgdsAyXwV*+veij|Ed!2qtJhWnX=>tKE4q?rBBTwpi37>Bt+y@(mi#8a -zYuP3u8=NEFvX)ON*v=T%CBvo(7;3&u(QUF^&}-JUvAqj$p}*_?Dv7K1!;lcHXyvQr -z<_GSr8bU3egV-dfds+jLYQ-GWasF3HtQx$xkb$ObwGu#*&G)J;i(<}Kk*tf%ceX&z -zSCQNiCWfZcw#BS+r(DoBR{-}q5)`w*zn72czjdQPttb^Sd6k-!-Nr?6`b%yUc6_Ai -zl3U30L7b=%#ftlGloZR3nz_xTQ=-iF3+|}-FGVeVz58qT@Xv1bclU%Tyn;=n&_B>1 -z^80<>6s_DZlK|l0*LslO)^C-4HXY5V94P`fv}`vuB3kJAE1`x=IP(ByR#<*%1g=fM -zK9QCe(L#GndG98xR^%5_Sv*7uce5%-ianI@YM7V$3830MNZmgQC3`m!-ePn^FwJ5r -z;j4|Qg0FQx-x71M?MJKZ!DN@{G0l)w_TuCb(c>n{D!USZp2-?%?x0|=>m{<*OQcyy -zH~#NQx?;+KlJlxnHtMTyaGPIrqfW4ggp#b01o_Mod_?58HHA#Y3mzx3%gIXIAWAbn -z70CLQ@|5$vG57L7v@UC;*h-D@sP^(`f5f9L=AaVpBcueRiymv+W0lqT+eMFC-?hpD -z{egbEJT?Wl%j3YC3{9kdo=yL@Y%(~@$>v__(g^f3p|D&n@kIVWB2mm``9otXQ6kxdk<7CqKd@ya$)oL2PsYn6rS(+^}jt^u7fzQ(XK08?*U -zQ}7&}(%T}irzwR?6aKmbR@qK}(-ESLW<P)jy2`EFrRaiy$00yJ*pamEU -zHJvKA$o-0y)NLFEN*9*{Kc3r!?20e+4|p($9y7>ewemRDj{ha=G^2eTDY%nv8OUp6 -z$`8CxwCyJM(wHme2C_C#8^-{tw0k+R7gpK4oY)Ev!uX;P&B1aQ1J<7@x_V)`3(}@J -ze=T=G+NwY5YQ1%}g;i>AU3KBX<`q;0eqf`&riPH6`w_TCA=0IEH;R}3QES9oYoxHs -z<*nr^JZJ;zVb=5IHYk_&Picq^WIZ;f%i_amrJ4}2Tkq~s@Z*dO-{suUIj81nMS`fE -zc){M8?%Vv*U5XV+2Kh2@h`Y0sw4!Bd(Q?{%1ee|9%d5Ze&5;Xefr}1&XYGXJ_>|Wl -zB}4_nl0f(#;v(*ught$IB^1-J(wHJwm?zz~Ue&!ro;i*L5QLd@L~;3Do9Or2;v -zIa$SWE9XTh6E)2YOu_ZleX@HYw<5rB_AY2v;a+Y*5Z)Y)Ma@`cIr~8P`h=*gg$1j= -zvWY-uMIo&=x&MXOiNMcAeAz_MfR}Lb)W8aSeLr5^KVd{90^Si2PA-%cl&q+92n%E>qptdM@H<5*TOQ?w>y&nqnJv|G)L$QjU*=mufu0$;t -zq14KN%pYJ>Pz)^ZtYJ!X&Iip$T^&toYiqaDIOH)gG3xd#b5elZS-_+^F -zHa-gN{)T3b-t^lY&bDQ{#lNsi2mV<`(aC0kS;KXAuq!|+=N;SW+jOkQS_d2Wc`ZAA -z+qd_?&J)6AM|%K$!$#F1Vnj3_f+HK%F3|!AxHft{OR<(2N~w+j>!WB8_S~?Lc*`qG -zaOJx`Lf;#3oB>|W^TPILduk^b2PGEWA!(h}RA-w%(0debK27Not^O4!BI#xc#1ZHopdU*X?oqQzja-C>4#;KT>6ybV*ydMsW+SJcP#s3UeO -zT(LE{P8T36%*#*jLF^TqdeRp4B-j?TNB3OHhhm0E%|+O4XY_`p+eCuevZzoVe2Q4L -zWr(_UXqGrUsdj~tOtOmv;LJU~0_`)IZixR93S|pIkv^}-9Na@Y%H5}D*OXn182Nm6 -zn=*s;<9IDen*Vj0nuyS+dgtNITt-=Lk8Yo)sze8eO%t)=84O5kW-~GwjLZX!%rCIw -zRgB`#uuKD1T#7Zz8LU?^a6@FifMw3cSgRSVSd0~o!89;VNvgFOf8x6@fM?x!iVa6n -z-}})%_HAoZcj879n?AX5uVd_FqFr!~wE3+6iHKTJZNkpAVz#egtv|4(^;*cbC}j

~`INC`MBVHNN{5=*Ldp4rD6@z2Ah1KuknUNT@U5~q>^!)-w -zAfrM6vJn-$A}S`VVBV%s-`jp@X49FgHMq5s1RC_v`qkpB0L(554{_M#;RSC5Lz(IN -zudnvuaBopb`(8n^=yJte%y^hxA=oAtG_%1)+SKBuLyK_c5C!qy{-B#V;8ny0`Wl?| -z33diwv>A>rbVq<^fc}G63N1CABv1IE7)S%F`3!FVjDn3>`!83|;%!Ix7m)Apmsed8 -zvu;h~$)9<_Dy?dP4sEh*g>ua;f9+A?wC@XuFUCi&JQba8J)@tfFzWo&fM0)2WN -zYqwwRwqJ!ByKPn0IFImqTPHvrgC6tts(CY3 -zFd)`9Ts1%Mt>P}5Z4)iFi?WJFoel}~%qCwvMnjJ2EmzG?drKMrHoN%_r8;)?HeQd@ -z5aaGen7;C=Io~_v#&>6zy?+&bT7#d6IKai5pT<}w@B`G~=h-?p;XGe{Hlq0{IDayJ -z%r3o*f*-C1#=ldtCE_#b7qvBbQ3LH^$6MZgIdplS;fy6@sNZnm5ND_5wtYk{D1IC> -zK0mpZK2uYzysDnAQ7QxO?=XJIz8_<}%_|6wyJ!jJT&SH4H%-nFWUKB>r;UaoTizKnv!^+`1^yD!YVnA5lI|KfQP -z@Atp`GPv>JyaRfmYuX~1chh~SP5>3(ionWfl-fkm9ehw -zrI)C(`QBf!Zrmi}xcx*#b0-BS%%3D7n99cHB=anTe1t)aoq2z7ISR2nK_A?zBChHuT^*UO(t$G<0N9}F -z{?g^_r^7DAP4Rl+@%;9$U2xJr0hG-rvBoor-BYgKw)IW)+_m0G>sPtYvCpVV(sbv_ -zTT<}MFDMq5)+{0bTOOum-Y*E60#GYX(@-luoTC51r8Ce#8x%K-2ICNHg7$A)E7-P{ -z-_E>>-aLc%KMk|sEJ$+E{o-4cO@E_o)K!9qPV+9=cOr^Mz=l*G#=Nt@eA@+->`^MO -ztpP#sgWC&KHF*1PF4%0<;!Pv2W9LxC?x>w?^9L@kE(DJ+2>Q8%?m@b)B}&zMj<4G3 -z8qp~|aB95|z=qM+e*RX|IhO`pKIh1@w54gGsoEoE9oynt_Pt9}MP@RvG6w{}z(?oR -z<+9;Ummy+Scc<%Cp|AOZ3!FqyqUmRswUY0-!v{E(zaH(!FGUoer799oFdSS)r7~bt -zVrK(&uVlukVN@1_C&xP~O^vShI+rC@1+Hwm{mm+UwhN8{APmUC_!h7grkTcT32V3| -z2`B&yl)~EpI1gUPuSswpdG!Up|z^8&yAm%*}5@8hb;7I&aZs@Ws -z?ap+x<+$ZiM9aY`!}YE{9NaDaxHx#St}SE1^KGqG)(6yr!RGfUG}Ir#s-i%SSY=eQ -z3VV5l3;hKZpd`>Ja0Ao2mk@67O~}b93#5T{MVGf9blrN9(6j=As!A7l&f5;b<_Pi< -zHYlCnu0ub7a;?BaCwVF%xrN}AO@)Dm39y$gv-!R~TIFmcboC^--Nts#kQ~c1tXHAu -z^>bauCuX<5?mC0N!M(5yFMb9?f7U&L7&mG+wUgD$@|dC69pcNc2IE=Wre+07M`q -za~_5W@l$N!Pd>srpWAr!Te-FIGPaMnf=|AubM6uB6;y_oo=n&{#h}fp3=gR@XVr%*OubNyRlE6n~@+7xUl&$ -zU}TT3f`c)MLtvMSz3?4$IuAR&x*Jl8f1zB!(??-jeB|-~7C3fFj!nm#*I}O=#53Q* -zyaz1W=qjz8R!#EggK~Ki1Z>fIbv>nQ<>&1oz~8BCCDosLqbefL8_iP^(pG-U73BAnULX=CQZlFpsv%6WPu=NX$kO47iGYs#R(=we$eU~|u6tF=$d -z0tv15nGa4dy|t_0{=BkIV30>aG$Hz+#Izl1KvYsg81Dy2Y6&-sFR3FG;X=u$gyQ!A -z$&n=SOg6ArtfoHM|0asm@98!#?MC;F6J`%*b@K-VYrxONA3o9z1e=a@AIDuU*yvqS -z0BlQ6;Kj|<8qO9UR*AQV^!$SNR-Kye_w1qwaF$u*k+c)cJDF_JRaWa^aHv8>NG~p+ -zMyf}TeU2CJpw!>}E<59BewEUGcSn}$0D9Xg+u`XgA#ehPLAx+}H9tq%&)=7=`B`%{ -zQa1Vz)$=qzBhzmhz^tN3Lp`N~!%G9P6;4Va`l&anC-S^ejSdb7`PtquU2DTly1$t; -z%ficC-bvZLHjVYLO)h~gy6_z0T^2f!>)fdDqY3mavDrUpOoz3iNKXR@2&t7vvQj8_ -z9N#7q>sqa{!B(Im8)yZ(dPO(*9AQ4#S73`7=!i6raInNTX6aw(HluEIO1gq@_=ta- -zrGKv5{F@WCO4A7kAs(3p1Yp9uM01Od!|^4k8p#m9=!)|iEr6bzW4dQ|rP5XN` -z7$OY;eqkG^wzCiYj`=Ja*c4nIb@Rg7gd7oxh$G(?OrD;e^J;p|4z(W3dyU9@pMVPt -zo;raO+>n)|<~36J8o4cJd%B#YFLXdri=NvQ(QV7wZpqmZ@+9jHy$6N+FTwm0{KAHf -zS`jH+#VQEOvxriP21jZsoihNuge$O%jIEE3hc7W8Uc=Dl>;TOVSKv%m0{U0fNv{J< -zgvmmC<{`n^ASgODwlci@s9lH9%BRSjYhgsl5 -zE`bG03G%wZm*!@MZ$Z!-Zu=zKjg7R2F<9QUIotIbr@}GI{5L0DBq-It>oj)Gg1IF% -zZP$HDLJP*>$!~DQgRCuFqcva+SKLNdhoVSd>vUh5rPFPfg45SZfYUawQjpy$*b4a9 -zk$I*h@DRf!ppTJ2pmKI3jIo-K^J>#SoFSw(su9@vS>3Y?mc=-$@C8DH6Pi16V8jZH+>BjNMW -zyg}ic(?(iHg?}P}X0vy%=8E_LL_{5NU@rrHfmBH4NJK?UqfLO>a$a59a7Un(Pg4tU -zfV|o9kWwo=mk^dtrDWS`;ob4YvVbDNuSU*wX@&KLba6Rv}gACrM=7mlm -z4#z9-)p-tYi_ocSFqddfQ77DxFg4OQ7r?NB`2rqq@9I2fIK@fJ1U*HQVdi94@&Vwj -z6p{UK`JOWz;T+4_I!a#{fLg=HVcESpm5zoQ_A!+~o3cVW*aXR6An%=x61H<-+Q!sAk^-hCmj_dQUAq&LClCvFg -zLn;9q7|phBLI-OkJ@=AbpT9*{b`cu5`|=}w#QiY+GYZnXWlrxDO}6FiV0}p`1bn#E -zW}Ph@V7d8h;XW1$D)67o>zBP4pw;G#oP8kslMwb8-%u+QmE89V*YtN3#Y9~W_ -z8^xmU=89V=us}dWS_^p%%7zHAM1bMe`Cj&5kbuMHmnfGgo*vyJ2059z84f3!dmt%c -zX=6xbJppOZlb8l5kg1@$#+HcF7^)p*07Ujb_LIOc3GUO_3g{fi_3q3$V8o%OK22Gd69imt=$m0>^U -z2+Vaj0{I%&c~UKpFY2IUeS9feymu=*r4nXbTg -z0_s7x!8rgaAn|mszoB^F$d(Ptf(8Mgf|+ow+4vca73&L1_VJ)>=OBQ0ifaCxvIC^c -zuS2YXg7qCd3uY>_j^bUHy$Khj-6ts4h*!^4;8QmG>91TdmtH}q6cw{dV~}5%rvhhh -zoS?sS1=b6=&vjsyY~QH&zI7VbRCEP8tc}&v-_3gS`*hSY6SR49g3ie<^wD$>vV{|m -zLEsu8`)ooB1a1)4S2f^fyAbSa=Hp9}%(>lM$!2cAi0Ti~8fLSL+PA -zv3UmZ?^K1Ku#U^#7-6IDKqXomh+~63x(-6PTp--!2Pwx2->}{%bc4jLkG7)((Cwml -ziwG=5fJ*`w>tI1Qxd^G2FvSy66=5wAm=ft#p>0S2w~M4w5NT2gkB2h^ng)Tifs{0W -z#wM#{iIzITF$p#{W%?_o$TX8O2~K-8^no6{P$S7Qk_>=tVNmfoN_Y?4LDoZXs-=n9 -z9p)J^V0JwQ3%YRLEnv?jN&?Q)MakB8327SPj3=O|u7QBotd#eJrlS{;i9alo=Yd1e~|Ly}}w_%h^%kJ$96|O -z_rfN2JG#Q@nj|(wO#(0UH&*cS;mG06}Yp6?7rEhRF)}~H`MZlP8x;c3)F`b?DBD?k=;t3!$^S&h9%~ -zXgD|hSk-T+%G}*$EV++%nVntYlKTb&=7z6MsVKScm4|#xb)k#grc=IVDwzTT4t(xq -z?1;{O34_pLdRa~nz~CA4ddb#EGB1NKiKGt{D(DQ+DlFUkMhghRxa5D^Mds-8a!sj! -z$$I;tti?BVt@ygD=ti7RiMB?4&(+#S#4kB1x`tZ3bz~Z*|E%ksNr6f)i{>L;6~VAPLuO^L@)0{U -zKYD$aI_9OZE@@-7ssk}g%*zLSrSy6a5)*>iV*X+&++KRb9<%KN&&Lq+suN+ViY}*@ -zZrOpZ+ktQu5sKakbb9FmYQ`S{HO0su_XJk_r7QohT`%KZDWYXWJyRC^R6S#<*;s}N -zS1+$&m`l2v_I6dz$BW;?su%bv=Au(E;{E`4zTW|!&h%OO!Ybn7tqLjmmtT^Fj}WvQ`lMOXFTGl$o8RX;A$&+DoXlq7(a -zFR5g}g@FSVc=08wDXHtY$O^kRQRdH7`*U3vg7M;?s6K(yNeKUXLzUQdTSWWTM9tq* -zWzTeJs;siC-)a($3#4FrH-L5pBi1UsJA7OOiW6vjOEoR)YF`Gd;H;Tg^}?xWQeQ%e -zU@NTfrG)ZUxo5~Kocil+(N-_b6O<~wtNQq}Wp^FN1vLq#?}@Fg3~-{g1o66WIxdRn -z6GQ0th#rGw49;$pC4zUR@hU_XNT4x3E@(R8FgXag!f8DR7ME4{e1G?sk1DbZ)NpJU -zxFu!N`e4$OBvg3sNxVmF1!Z4i6;@n3F0vUgZ|7MDsCH3Tw+O(OR6iCDgLj7z{;fwp -zt;<{jS5C^-JHTfh<{by)v{`T!#vIgTI%$=??qa=!HBIS)2RpJ;-NyyDHgPYCV!cP5 -zz@7Fmc00kOr2Q1$E3sDL3s)*chQpXwLRh;fGqlLpY5U;3<34MY*xPta8Asu2k -z9aiC%&Jxk^CPyAU{(WFA@%!b)o2jxm&qOyj*c5X)P< -z;O&qx$P?Rkx!$ -z1?l%cBErg`vH_a!+Prddj&JH9bPmCVP~Se06g(h+%kW$W+!T|7{Rd-8S%uX3vIGFw -z3vY1}W&YklW82XB9}*bRDS-gg)_|J_2>g1sM5NUzy#(NOiur3t)7OsbhG(i5Ot?s^ -zita{-{m4l*9nJa^Fr(l_7Zn#3JVAfVVKHFnka0AwAegVW -zK$dvIEWMddRiIf$cU&`~%06?LKL_}utLO91h`g7n03R0BT{nhL6NpNXmzCRsJ2}M| -zWtNWh=j;ElKiPNrASTWd2HlxyjTNG7TD+TwN6S8NsC86T)$e;_+uwI+n$o&|?bMlw -zwtE6>n$*-+-qxzKGYCuUTbM)d^j>tfY<>&&_~LC?>FOHBae?9`tVSfEhs|QV6jvi+ -zwT#w?<^~frxI!#+Z^r1=&TCWTzv%#=p3Q81B71pc)OJ)h&{>kB$pwV@dvhRXRp!_l -zLBIVLKcMulI1qiE%)HH^jZuNK48qz%6(~G)4#&uh_1ZK04ena1COy8;QF<2>KPux=Rbf_qr)3L1+Vb-8;lt-XdB|u -zm$+skKQP_z!TT=y%_Qvib1v-2&@XbJO(<0Cj59y!K+EE|`gDi*O5AdIfCTTU;L`FE -zc%O1R1den@_=?A3V59X|3_Q-l&vSnvmR&afH5jGCLh?i+9I4E7-wZ)9#$WmF`7!VU -zZ$S(&k1rcglm^p=-B_FYxUuFg#hGV}H8;bdxEQyFZqCPzf*!dGA|3EVp*@an-RObV -z6$53I4S+IT%gKfyu2WHGkIbr$Nmj%7Tw+5JOYYn`6$h~=T@#XrQAR?%&X+i}`Zr8_6? -z3wdatnt$&^tQ6WG-*ltXFjEBv7K9C(RPr2Du}z*cLGxi`>Bgv?1hjuYp~zhjF|A@7 -zqzTVXG@QL7teQyQ*5tXb@=>9?lMg0+IJS73H_hmRG-C(f%@Qo+7R307Cr&EHF!AMC -zuulns6Zv{3*l(cQ7ICE?Mji=@+BuEBZqmVx3VJ$fMs`%z@~BlyqBbtlU+6T>gWIab -zaOGmP@n>k_$fdE-cXz<(!x?$U*h{1iK3W3j^k>o$CmYU}(q*e|Ara!SZc->G^8sbN -zJ8?PJcyD4Ox1LZ`2|`bbH-{D^ve7x?rqE&nD|nh6y2M==+RVTTo*+W!DGEc~Z~0$L -z&vx)Hr59hJ3U8Z~KwI;xA4JNiEH#_?oO|GdVcQ5-{#VTgrzuINM788geB)$)S -z0^$(2Y*%MVP)%w`39qg;RjGVtZ_p#+V=}p7TW7_b#Xbd>0!k)Vh(k(3Dl5fwvB}18 -z4$RMx+S)nUr%kDwXFPx0C#1n-dgcT?8iRQXPxb0?ay{n0N9prBYnCW3PbUAB4`7+h -zzDY`NfH^&tqiCFk`AEy2?NmM6S<6u>&Fl>`T5pI?&05IarhDJmFiQg0q>lX#jt>$$ -zwa={W0O#DFI>oxUg3(p}8C@Y`#WzU^#70@@!}&oZH54NNLL?&V$9-BwI191 -zgt6rwTB0~}PcNdiAGpC{RHTV^cYqsg%#sFO43yzqF8=)vD3TK~x+*+-dQz!PCG1q@ -z@>hk^iEBrr{2q%E%|Dfn=*!vKwG_P^K*$jFQ6pZvqTa_Q;3_Cz%qY+qNaIaPc$y72 -zN`uDA!hE8)vT#(x@U=wTKf$PHiMNCPJXE($fJOPOf*f1KgL+mYTXD}9`$V9tCKba} -zi{gZE-M##qG*`B$56`B-uO;f=PUPn>0_%OmTWJD?Jh;{4D_ef6-lq|V%232m@KN&> -z3e6}QfghTt_^z*E1>Q%m?yj{Diu>t(rM|nvT^U37I}JO}Y{d1B4&z~X6a5Hd{iO~u -z+(-wm$K*2p=ML4+9drGr?o)qtJP5%sx2`Muv7<`+`w0U1d2c(BF|g@I0dP-&rhm4> -znD{UB;#)fAiN3Nhyo)I08*I;*Pj{gE^z&SCag5&9VcyVT+(UEZU3bcS>0ZPB9PSNj -zz9clw1e|O$YYEK(qS8l2(3$G#O69*1l`Q37iM{Mc^;8e&!!^j->lc5E4#JbPRJs=r -z+pHaHbE5mr7=3kzdzZIC$Sal8e}0ATZ@7o9mM+prvoZ=Y1|cjGDdRuvAj2a|)lq(| -zDAAZ-f+np*fs*BfB$*gG>Dd6dPke=bRuUX9)487=3z&<#Wt*l&{E-(MvnTe~xJxz{S^Ly9(oJ-a)rv -znW?dT{uVeY^=mIfXEk09Y@bn(LR#UBlRi!muY1&ApnaT>v=Husn5GS07G;;)*I%~9 -zYRvM>)m%A%*>fHXJ*cvIdf(J -zmv1R9i4>Ouc3i|uh`_}t2H7Y3aiQa5NdNeF49(^nZJhFJDu2UVDmjL8cYL!+F{S3| -zP*9he<0{PwX7ku)^TZx=VxM^me;Ve5bYK7nc7dmMOEl@&6sRTA3wVZf)lD)-=`@s{E{N#)iacLd#Bw%~;C=Jb)8kTMH=)$6Kxr=`& -zrdq-Eb_vs?Ol~yl$1Z*njrySr5-MO%bahpR6>?o&XgZ$~)saKw$LW2H%CKVYn=XE8 -z)UBc(Pb;nzQsar?UBpnl!%wHSJEaN>u!qTqS`u57NT -z0fl)(7J^hW>sj-Q>_sSf&QO+e@COhcLz~aVcA3LCQ(b3+7cR(0$C#YE%Hey7({(Y? -zF_*eJtzPHObk=-Hk0Z~bAd#)WQ?$@-8O#G}8gXQgB=zEBp15`iLRdo+kys;}E3%BS -z1}x~wz0|c>4DxYN98A@e41Txbmme9$ZRx6lWf~OA$b_=cCX_`f8bc9GM(5;qQCkK^ -z6Eo_I&LW%)!%Sf6va+1HQ=kU$Dx9J&`s~ZXDW2$>og0fj`*!Y|)JVV{v4Ew%8}``% -zTq%40&$(|v)Fh}QS;?C;*xm`=w8I#po5;=vtS4{3C2H`i4t(- -zSXn-bJkMR|&XN+cU97W~Z&Wf(EVG0C`c&YoAGsR@&{HnzqOMSt39f|EPt@_B99H -z52-qsB{Ahv)k-CCA*y`uqx+p+IAP=DU92LCbLkS? -zqqv?<-aU#=KNmX1<#`B0=%fVMJEOu@m9y<`XIVyJv8QodZrpxZdCtl2q%V&4JRF#u -zY@WWetIS;0EF7?v^;Grod!ga?tkifhB{qVL46P -z=t8S+?8H6u5Z)>3`nz6*@2Gc{E03sV1>nYyyr-Ixs)fEt(;C`fP=U+ov`4bPMskME%Fhz%;&wSQkRw+P1 -zSLousjIZY6J~P&ve5vXQ=!n_Hw)e(6Z+oeed#Uq%pW%Cb#+WiXPh@l+_8E85C+Xap -z&h}!@YU;vegDuYqjw`X_=(G`PJ)ij-VoPk|7ZuH_WvH6H9BKK(hRf({ml9kw34ml4_#^wG;fbANZ20O+iEV@6f(70uDXMlT3IV81eX`BJUxE -zyOInW7%^7R#KE%)(zi02IB_q^TpbFl%?vl}m!jb%gQm@5)qrxy!&5sWft9$nh9-B? -z8+GD$c4Fs&BCj3!Ic~!ZC0^jSZdc82n>wn71lr|e3P^DsLwDQ3!YMuRCsHF755@c( -z1}ZhCU>o#VJHqxaEGHO(a#p2p(P0yX@NoPCGOlRub-BhuAbYx;TUs3VY)& -z_h$!%WxlY-9AC!Wv_~B52XW^5#2BHH7Izij{9zPV&0X#y4z% -zUKKLF*>!(le&~9a3%dy_ccQbxcoVB&sn6KWS&s>M`PLtwNbw`D=V@TzhJ5|cF -zxP@}O=J0VR0pij0ax5%DnSCjy&n!ogkRv|EIWn%n@ay3rvCzA`V1iia-l+a9aA;)4dI``FX9g3 -z!aNh>!4HqZuNCLQ>^Kj4>gU0|+Clze%k*4khbgmz-z$E6(azvwV@ELL6lnK~kXg8) -z8PESjk!iy} -z_BA(VXB7KU)Wj$@3)nJ`c<_7_v$tOc2iQhLL$SS-q8ubFR+90|PlS7-!?dlP->K)G -z>cII5ze~?8>IhacAL+S;sG9#!4+!MK?_)rLraz8&2EUI{e0S;%njzf$4qUn-B!+!c -zFPBNUNA1B$>>GM;VUPzm3q`W;>G@hEU#DbeC=Hqs+$383dwOmnie>OSp}%EJw%+0; -zF?f&Q5@_S!(Q~6|YQHlxz&jW+Rn`R3D+t18!QFy&gH-4Y*g*Z{eT7DkCEM%>bQ?%(t;H -zC<@DRd0q)(eoX%zi&XAb1-e((-c!MIP3>LPoq*=rJF1uwJ+*(S?lFC}x6>XS!hH^W -zgbY1b3H)g9vd`$b1MT)9Fp2+%o;!#J2rufnk13+~-P=A9yD8!!JO$Phek0s|d;2Hx -zPwKf{?WU|Y+|lt67U{Vi?WUGCezl$}X*X?e$9VJL7wYA~67D_J$*j_I@7nA1V4l!( -z#g00kW*6wW_faRmLeFicU4`E*jyn0rDZ5`K+}n;i*~j!;R=a7bSSLSU&%N4iI@HEK -ztLI*6H)YZ~L)cV(g){RRVf;Z;#uxT?+{?6nR?q#jzy7(H1_tVXRL`xW^)JzLD`@@r -zUD;p%9IylbJ>g!k*B`^q)^m&7O_S~Q&(d>Gx0~K;qcnW7-SiZ#KZc#D7v7S<0pj{U -zFk#gZZeBa}kVNM2)Aih=?WRZDah+_EFioF#-p0+P+yOd1>cEUm(Q|Y9G2^G|xv8`l -z7V5ba2S)4^iqY4Eo9W;}b}|NhyJ-@wdlH|l=Y5nZU)s3ww3{E-bK@KZ3yFGeOn<=) -z0V!BMaPbj>N#&z$%BAC{nQM9!sEwUruaYP^X}d{Et9)7*t+%{szk^BV9AR@LGKyr5VdH5&k!L6IqcANv -z$S=g97=DC(%9evSfQ+k>yV_>@>%5Sp=We%|Znu&B!EZ6KdhX{o3<6O0S{wQYzgOGP -z=43tB+cq0220eEf@ibE};@KoU`zx$q*lRdMf>>&?u`89#b6`AiP8=)6>7XG{X$W)U -z&b65oZNhj=7i}VfF>HiBKi=RkFVj>>g^~0w*)S?o_A1z1S-}>Rd4)?t)uV}&R<-nZG$pZ;Nhdu1J1co73QvQe4L)Wq+lhM$_CZ!AZP(5>3fp4tSp?dB$`m53NeG1_^=r5GPT8^!7=UzcPros@0=(+XwZx2?fXRa&Q -zpA{w29^gFO4c4NnFI`1)&LR0y!&rLgF;U5!1*<UindbRqiu9A!N~O_$awY!;h#ud@ -z+>Hhvrco5!X(0ZgHvX?@KDuAV(`I-UG;7n9+IXT(_TXMS&%W$IDuG@G{&FxF(65GJ -zz6Ng>gU^r@9QgUTVXMH!vXq)L$D<_}CO#TDp=9?&B*f6b+w|41J?%VVHGeCvwKF`o!A&zDl71<5KRLh>=!5f^V}!l~u{wpm=KzQD}ps -z1wSPap9xNs@?#^XYbg4{_mRPigntE!P0@(EOnw2jf+R5SH(T&+_!vQ*qmG`(uB6B3 -z5dO#e9K1M7cJV3M#m%yd?}cCFWfu=jxp-Ulr}+qsE4TM;E?pQu!#l1{7I!wGQ`3+f -zH%k_`NEVmx8~45}?qgZp7qTNzUv1CWoU^cKM&-q8vYvF$xQXH7F^;(Z^d6zrgldz+ -zw5g0XKB6E)o9@niaGsyzA)eoCKaWG4|N2)#n6dQE=vi!UZD -zUOWZ1URhb9A?|kU@znW!#&w=qS>aHB+8xG}q;fB`0tgPnaBt&zci4Jl8AQG4I?w#{ -zq6%A~0q`=w;|3%xptO({AYQ{VsPi+5_TwQNSs_%R>u~YI6MbXO%JQYPicyQVdqEeS -zg)-K878=ys=C|e-*-KnJkj1+s9!1QBx(B%o>g-9aP)#FCekF1k4W30BS|Jhvf?DE$ -zDQcE(d-6OQl89_TXq_7UlkxaL{7--UP+Z~^d?Y_!Wv9!i)bpCjR9Xj(LLWV^kvhIh -zG&?tFR}9l`@Yc=<(vI`jE_C6hV;vI7Eu0@|*I?ktfPYs(0%8Xg|F*&b -z#Z@1=t%*5r`lf{n4QypJ;}i+g5hpsjq)+YOm%|Ua!Dqw;GZ%i|^ivC#tq=wmJn!jZ -z6pT$L0_|fB)LA)jVk+$ARw(W`8@yvVzw@T<7OLkI_D6HW&Wj2PO>sYS)hH_aLr+za{)=Bhk<&1am*P3VWlu -zYprzlX7@%DqJ+gRywbWAPRKQmE#`h~75HeHI7h*K-^%dOPb&;QN`tRk<0874@Fq8A -zm%PVwT^E&#w4}bA+YV1~$LwevjNjx6iSCvDOmB$WHgLQHo2SusM?z4Itz~VxR -z@Mjj?l{-a})g;3hJD@Y$-)hQerTxwqMsww@rgE`>w!)y{)DfTB+R3Ai%epmQyef -z=3miVX{+fY3g*GS70rDl1_){53!=G_R?|+pS~Lk;qq!ZerqmXAIOHz~Tg0SC?I8xC -z!dALMqlC@T+*_^K9fA!!*!*a2OKYW(I^Fc(H$`&=t)^t;0fXexUIks*7y`~78wrtP -z6qb25p1?$FG~r%tHEje+{N`vLp^dGkNx+uRk7g1G_s>?-L^MR$6wM;^Vyo#vq@hIG -zdk~-G&JqkRlMWT}Z$=ZT@Bpd+B-W95!adt+dKN8W-iYQOfWMDc(=r;$jRq8#-f9X& -zE;c6`4S)u$fLqkcW=ETbwTLHcPqy+|aM?r1Bn9_atMEEFQL@L&$g6>{q}0o}Tg^Ox -zHt=B(_-Nad@i&($M$K)7moa-!JM^7Ytm2Ne2(O?3=5@thZ^C`iLL>Q&(M+a-?o>Hb -zi?9L7LXMdHX$y@M)ufubu{-IV)Dl=G*Wmun#ooO8x)OQ?CXlgEo_dW -zaXkB`q9BSc1rA%Rk}&PK$F(z3cXn!51Zg+;Gu4{ -zDwJBOo_>d?s$9k%1)y+rFrX0XU-ldOkaw9gyA)d2lHV$kV}dW0MSWEb?(AWjEN7 -zI|BcBcanF`#=j9s9X>iUBO$GnISV5pLy&+*^q+BO?$u8I6FOt=AB?-;?Vnd`o9+xX -zv5X%R37>mZ;SO<2{%>1ZSd}yD1#v)Vwy9)v4w)siQmxA@p$9*ZHSj$_XrWOP_%^y# -z_oq0@BQ2le-VZzukBw9X_7`X=xA7YyQ_5`2ZeYRh9*hOA@P~miT4vU%cHsDyJjtIE -zOJ%=R!kUFK(i}!zx2zgge)jHk^@kf(%8xp!)?1_1=(Xt@ZL(gQIx^XOcT{-`DNPK^ -z4W1I0;H3^T2WhA~yk&G%Dkb4?mZI6+E5Bt6HMZp4w1qBh#yNw1pO<6saSryz8xopP -zJ6dk$xPr-b(maU-@T1;uag^kkx19~?Lvm{G+|;>o+ggmfjiphq^D+z);Kco_#q=*a -zC3x^tqq(gurb*3MfjopMj`@Lq#vM-+nBHu`DH7pwTiD6b%+v0s(ai|o;a+cHlL3fw -zh>L2nJYM622N>okglF9gW&(^~6Ac@%{qwax^K+ml>Qxv&&MYC=0T%0{HDVqPy-6EH8Q7xvBX6z>>MzgK%G!%f4Fd>@% -z6{KeAw@9p5$_Df++? -z5dZ0$bCi~oTIiui9rK8e8ACZv$&AGho27%Mw@}G^Xk&**GOyxI5_`XB<9)Qr5Fewo -zscP)tUMbkeJOD8VE>I?<+_7=>=a_PI5;+A2i??j>yZ)qwI$dX0V+|%<7QD!TI@q1`NVWb(XM3$01cyE8u$%%=Vys9X7 -zGRCdl`4YcrBnI53k?NWoB -z1H9wG$)cBpQ7Forcg_?rn{c**Q!G)GOX24BEgdat7FR9hkBUXHTXY?l;HP7iyti4q -z8lE~(Fmc&`gp4pgle>S${;*(Gdit<*jc;gA|mV?VyoD1ymsE^rHXv% -z-V%Sd+pD~WEkd75?Bp)ew%UZIjAdyf7* -zbd~U{JMAhqL#&K;mGF@r`UPEHHkPLHK7tokgjM$HR@3TIa%|?%VO7$aV>M|%{Zip# -z`M0gYi}1CPJ7baIU7o&|KJ@RqUnC -zT3a|)jfXpq#KHb`-#91c3G}(JU}PM)UfJU$0o&h|>Sr?h{p@3OiG-WgT%j!wxzzYP`>|hqx<-7%(-#BL5CE>z(F{H*Seqxq37#R~gmA>&5SUP=;e8{)S78i*S2}v2gnRqxY4-_I4{H&2=bZL4R3dy^K2`P+mfjUiI(hPBYY%XHu -z7sNB^z-E*AtSJ8d9K|1U$L_kus~Wm9=1X^;Q0)eZyoOn$NkKgH8-)-2_%Emu?Nb~% -zi))VQb7bZe&1`SfBt=0NJ7uJLOAYZJQ-zluF}>hVoY$P#P>J;-5&+O~x -z&h9GQLMNHLnG#E!lbmr%Q9I=YJIUovDN-l7O(OqGlJ|>5{=LL -zLnu6NwS>EPPI!#2^{XYp;i{vul6cgsg4fy|dqa}66%UcW4u_0e7xEyUPzf2Ot&W`* -z5`8E}$H*eff8}M^)>ZvlYFx!^^=tfs_m3%937XA~Obc{_vHA#jQS;SS=*hhwLn79N -zgp*(X2pOTR4xSbga3}`v)bKy(3Xb9@IFLpp?^Z}Fo1H387c%G2 -z^--0YXy!3pj>dP4zeb3q#9{7W{bTRws%2EIVxNnI9KTW{%ky9#m>NS`8b>@JY?FXe -zOq3p_r*0cNbK%qjcUqZSIy_6k-_liv`gcfs!Fjim%TmjoO47G`tE{34Vgy^fA%c# -zzW%S=vmNT*Qs^Qwzb)*@O{`Ya*8r+>JIpnwJ;`ut9dbOka!G*j3vI%6G@(h+92%fs -zB)seg^2s7WhB=D%Ks=Nrbl*cDvAQO0f=b0e3Schj={zfpibRJqUrOMlNcolO>g?vP -z?xpSej%{>PUQs>tNZOSiKq>k4Hu+|o@R@u0C8pf{)}=WLBktjhyu|jR^}am)KTO!? -z-Yu&P^}VLR6jEwj!LJ4m!b)9!b6Uwaq@f7pT5l8byh@jNvAJGz6s4iKBUhv9x*&Xq -z>?-zq{P$x&77J0)LeK@vnU#Lk($Z$9(o+ZcH+5BgX+L!mIL-9UV{OeE)qaoOzQzyp -zn;V~Kd^mWAurz?Vp)WXvEl_l%xQf8+BeL`a9)?R09>6Ro*f7WW;-%=c8RN>%sZNb6 -z(+Hzz33og7DXX0VAP>1cxW?sv-V` -z({r(y%EM5hFjPklD)*%w4KHbi!PK#l=7v91PmS#8!{bfYa4TQ9eJ`ymHpSn@+o;gk -z>QT_zc(k{<@wHohjSuJdsMh1QCjU8F&g*of99;!pC}rMh-EFc+gq52qX -zXUyyzKlVdPrFxP17)2zs85-Zi+hIM)5CPPxmw@8!p+&x6C_Cv^@g&3RxUt>V>T&xqj9ACBY?y03D3+7X^a@m21YckZEG@aOaO4dD|yf9!Yk(i$P& -zZ;l-A0?@j)-UF^zbZIL~fs&6zb%jIua-9_j(Ra*5gi>ehMN!?9q4(~h#$ex2AS#kDzeeP(x<{asW>4!d -zA@BWAK*lCrA3i?G41xg9&fgcQZimt*1Trgh`pZ_P3LqKI+uC&pDn}nFrXv+y&SAm{X=20P6 -zfB{Y^9ycRJdB$K4;Zp?$(&?kC|gzY>lr>9dm40z)yfRw;zjRDB|c`oz1` -zc!2#)#jX8{9tU@LvlF7UVS!d}?d)Ov&RAVCOc~i>UOBsavvSOP$2v`Np~jh@$pZI< -z-Z&A0r#c0n=8m$%{jNT -za>2$lx#2arBVPPRZfT3WxW~UT-DskMGcbUHagQ6>SCv8bH+TmfJCcC|Wz`0(wTuNv -z!mJeq$Qx}ccsq>}>gXZyNyMPL*=@1qIA|!(;EitZc=~FmcZwN`D?Ta~%f!Ex?Ax)o -zQgQ43=0{R_vIIB7mv8o#AEf}gSOYa(<4?Mgh1k#!c7r#WDc=j6kicA^snryxQX8y&41p8CHfe5S)3MaPs$dT4 -zs$J>PU#YQ4_*zxr%m>6WEA{YDG+IWKMi_oa6@FY??wwMM1yNpvf0ja{o1%1Cb5|X? -z35=U5#SY?uB=XQ@Z(OxUnd@zsho(!P2wU#V;_-p?#_74L?`2xwr?ug)6ybdIPxsQg -zJE|zJ%DTT)TfNM6w^b9pwQF@;?-}M@C2{VMP7Co9c&IShkDT?Fws-%W9@}ZL6$%f&>Zp}B0 -z@T4I8Gt^$EOi5&xcANGAm!kSRX)Au`Z@WQMx^jc8h1U_&@iR%)XUvw{HM4Tr<A)Ea)$j_xx?$&zV^S&q -zzK%SWQZ0TH!T>62C|M>yV&iv93{gW+-ys_~@R4i4=X0C#cG_oGg(8d&wixG4&BmW` -zm1QV)gl@r$Haqc0RYV3CN%V!)aaYjVc@o0|)U>+eE}4?D$FVl8`Ku|lG%N9+x$7$B -z>pmB+^Vk*~)8hN@>|w8-E#Z}T=z}HlYUv~#Ghre9 -z%LX@DR4v56tvj@DS+{eA6SOENm>Af}s%E+*4QZ-!nii;ZeNMsI8gT?bc}daXb)u{3 -z(hFy11wR2;0Y1S+h(HF6M -zC}a=3h{^wIBR&=dltc)aOEcpdYuM|y<`bSBAwPu3M1yju>6A@UBpY`h;D5FSjaDh{ -z!goFL>Ld^KIkhDalwBYnjEH*^sXox$k`ZW}Z>30H7pLtyf!WAEB#ad@-O -zNyyRh&>B8KVW>K{ms&`R`}R>IX*pq$qaFKrF6In{zGFl;=XY-swu5XAB#M3&9kJ6L{S#(zMHY;k*@GmGhA#Rr=6TZfDkf)R?3^v;9)` -z3r(jN;SNQ!m$K*blFl^^qlh%?@gC~XW$OuOmTqxXqjo;3TO)UpRGC7H8iS7In~sdy -zG@;v~mo!c&2{l9plr6D_9^N4}1P?8ErDI8NX!|jz#)ohlLn7hw8rSWhj?!<1mMt-i -zaL-SYd;mWp>di@qcQ{+;HBKatf@SU!)$6xQOGlcWT%yiH5tQTPROBSvzHRe~%jcBc -z4cUYQbuUPeMjUmyl+j38Y)Fu1-G|bYuWgqSn|4004IZl1yBa2XF3*PIVfnT^C&}^? -z09$@8sPW>po-5y_KO#B#(J{+eT!I^SDgV-~q(9wY1n^8FbW}+x_O~v<`&(Rz3WrBq -zT`@l_cU>V3UC;<`Ofu7#O%?Jgm#b2Ul2p&Qn%-`T^qP@GTsJro*Rfu6T$lTdb$)j5Y^65e -z>51r|{LP^=Tg)Dkh&`m#0}yb}xNm=oT+%zLBb504SdF{!Wl^OHi9KE-sUX!ZLbglv -z6w`*LAY6A}yD<9M@Z%+;@}0`Og7UZSEBD&$^nS^E)06P}(mQ=02aq4extmru5k*xr -z`AQ{MP0H(rvS9YE2uZAYr4h%8&=3ai-F;1{p*1OPk-GzIqi6~Fh -zH?n87ti>ikl$(KevQR#XN-vBHZc&r5G>Nska0}@N(?B^N@wmVe~e*)FqXILNTi1LPCpTvnuahYQ-X -zaHP6j9iQqEpE{&6l@uihB*ir4hy~^>doR`714X4SdmsPfm!UwKKO4FTPoYOv>g85x -z9$uP?eNvGZko3eOp*g8u?87=PxXIw7Z}uKahWAN_6WR6aQ>3SQLd)LgqpcvNNAvC6MG+7IV@1b$9Dazv^o -zOZ{8HjbX`s~B&L3u2inl(zAW}5R*mg+%{8uuXeQAYsR;3V*&2JUT9~0zU?L1C%_(Q4TIeCSW -z@03+(ly?em9}QoeB`pd -z(%|>-rHIoV=@+}zo#{K*mgV&AA94LIJl#$?um#?b)PJV)$7N9Gwr4t(o3k&U%6iLl -zb9#leDv2RFxMTUiWjmA|P{II}%YlWZGS!w#V^(>Vd}iu&qS8X(=pB1ry{h?AzXln+#o+M!grmGYU28!}#4| -z)TmD`l?5x|SfhO@Og3vYYPi#w?QvG4rjr4k@4^}x&;OpshBq>)j=JUrH!cZoT#y^{ -z>>E*#4kk0Cp>VgSc3((bqOUf=Z%z5>7sEW)IlFGkRlc9=*7(>u@4<3nHl(0xXY&cq -z<`cd>C;WLI9eINI&RDwH2YwTsnWM7u$wkJ@z2hLqzgj+dM$3%vDx{{Zr;hwqxzw|A -zsaNGv@5xF -z!!T)l`dsi-rY>zf9EHv;GA@I^`}Dbc$4T>2B>Z0yn5S+>R69F%03MydT|bGvJ&e?0 -z&x9#pm+(DDOSu)VD7Idvy(hM2MQ!(5NROz=xYwM;*W#12dui-eztvD;*1t)#oNt58R|PepUzjfz=lsv1wHZrFe!Dr&wKUA<=6Kx2MAyyNr9Bpx#>A!F9!b*VUrJVa1V_HO -zA{T2Z^|e0ST1(sw!Sar!h7hM+Lwdq|iNY{kp8t};-wEDW0#q}h-Us7AV?rlw9N;@) -zdALRWhz2WGc_gmP#e<0`NEu>Xs%%WtM|IRTH{DCCfVLR9^kM^sqIE;q;mBqA%0_xu -zdCXm&o6=;1T4qLY*-O|f2S!we`C?g2+o~0_d7!fi8QOq1l!ZWKlbdbi3@TfloA;%t -zZ4N3~3PtmuVP&<*y(J8!@&I>$R7X$~c6olZE~wJ>qps(3Qz~uBuC%7W{FjvZX}8ox -zVN^@KLG2)LKY4v=p1n4*d6h@XVVm+)+Eam`uDc;dp1%~t*fj(=f;Q1|>q`bG)G2Mc -zu&otoGlbTU$s`5`C7{iFbo2eX>*bd&WBq0xhIadVR2IZf(8=Gnr5(l;2l;wDAW2%v -zR&*odC?-ZEu^V>hN+jmJVN1JlG413yC~L1nF5Pqxf6zwo9>!Rw`&vD -zPjHC4j-=qw8PQ{Ov`@NeEKiT7r~!$N7*bxasZM?^)ATf}|D1X~0k+fwG*RNuD@K!V}_1ltRD -z20=1+DdZu`RGVt*xdBMZc%TY%dk`e%GeEwVGQpeAA=x;KPcqOmN;AR -zgW#o+wqUs>#zua~!*f=alxQ3ISRQ2y)>?Enl9v}DuC<{{Pl@|>F8vhMS7zQl+Z>1U -zPciPAGghdlPmxUQ%=L~n65H#ABBXGQ#)#`G;vN^{_^ShPV%~XTUco-sBuT`$Ij1JV -zR*R2tlqbf`5aC#@BM!ArvB!jAWwXlxB3z=qys*J}C!PwXvZmoO0*CTMWp5qA6n2FC@7alHTFI6pCt^&1@L -zBi6#G9C4_3h&{$S-eq&EL^`H5>8u&AV@{9f4)O}K>Xj_j -zQ-MGaTX}bUiS-^zmSVNJ_`BEHJg>MyDtTKu65X49eUH}xI0fJ5S=i_mtc0h0(5K^E -z&0ZSi+3iKP9?18^9h#~WWF~#d!$#km%ilKJ-3WXCd{4bQ%m*MnQ8CDrvm0+eLF)Cq -z-l&GB{7WOOi;v-{FwaX@t#BJDdc>qIEbiaey_r08-kyf6WUob2E>5BQ$8m7g4LS5? -zve&2&8?rQB<24sGG@g1tpu2LUiR~}3xBvGDJMF}{cN!?4GdCP@=%d00@(LmB7X$P} -z+|~xF`hr5dzu!cId~Q<%Tjd}!m4n|9djgk;?n0Nwnnd9Kx?dCP=C2K|dt(C~6YSN& -zaW6Men@i@W!EqU4oN#$?+-fn7?{&m+&)K=)v;WB-%j_v-1GuZtCUA?zNM>(8j%edk -zVidIF_BJlTJ7!5wP%f`rmJ1&yE^E>n=q}3Ygr}iu5*oZR-LmpLm*b{~ItQ0X44!dW -zzEmbQ&~cK!EW1Sd;+vj@bL+AmrxB^ka_}GKj3Ld-vKW5_0zspp*I1TinEXYV(=1KU -zzH%qSQLdl925)83gob$&8sZ+y#i?NS_=b7o8`Mt}$2fE;V&9Fi_h*NL24LOM4QAu( -zAnExCI>GhRKGq&9oChG{X#1n=?SH?&{onrjWd&ZGh#CfO~Cp6s0Sz+hEQzFVVZ$YKjo6&0GwDDe) -zd#MVRlQ!_%MAXEPq(LaWAG&K5-WEgL9V<;{>>+Xdx~BVczgbN;tz=?z3ob+Ixa)vn -zcjzom67Cv4nGbY+fVpZlAw*$t^vqqR2_+)T5BL;HbP<60-fF_rsuYIL)p3_-!n-2O -zH~8e=(~W^0SBsSEu$sOG7@Mu*+Gs*CNW!(yuXpf;_OY;2Qf9(e4&Go5Iwoo2PD+)= -z0`*1F0eoCpu3?ryiF>gt?ksw$!760h^Klj{jc2m|i^sVWR?`V9X~eMEh&wJ^wbfLO -zAiF`w9krT{i7EVgM;TRCQgnPmQ^4o!&D*k^Kj@5Eld$RkBo -zaqZpf;E{QS)=~em66&-ydy_SpY07$o4yhcrX~`BVyusr;nQfv~EX@-CQ!U%Tw;hN7 -zh2}#p7Q|(3!QHF`<9@YB_8#&Vh>peQWh>7*==G9F0b#{|(|onnWUx~!oziio)s&7P -z4&J$Et)|6d3cF;0j!W%5w|Ib#$nuPpo=yn=7>J{b5zecF-S-D>AGU3wRqNxc_4U$* -z$7-jzY2S#=?AwmPzwI$lf_?xb6(A$EzItu=NbQs`?HgX(U8o<&ZF8(R#oIR9inl1U -zXIt@6YQASIDbABfw#{_p-ZsN(>DPsd@8@Y``=ho^wFbLw#??k2ZMf(0gW3e#6P@U& -z_F+dxy*oF_in=FR4L+_0P2?^GyjN($Blg#66UJ$Mry4Y#yA)9TZ+p;+!?$e@SaE!4 -zN6U`3L34ALf|_*4S+~VmE6w)~>?npw_la2RwpeSk`QF)`#X$R#-nvb1?K9uI{$VlX -zaXW8_O7-WlX~s!xh@=~)bkN}AZITF6n3eJNVuyO&*Co?7vBP8qJ1&J-uYA{o2XUp) -zFNP4LPf>xuHRmDVj& -zl4W9f&=A64=_VdYn&yKmb3s!eGgZr&TefM%csEL%e`vV^p?Lgtau6O-$hY5)$E}l; -z*@gA@@ethq3y)t!5N5pt55bkw|~&ZXm*j`^bhd(1=QOXqO#ckM~>GQ1EcGC1Psy7T2pI)(8g`{Ts% -ze(WTb%Y^3~^?~w->@mv+)&!JSd6Ihn5TXah3*tT%a=DwI9F>r|~aW87sc2W3$$iXIiJF_SD -z)5i3F)5eiUY;*2OD(CP|IofQ>K1p_Rub*TcTN4KjH=Z=Tf&@OepI+ShlT;Gxma;)Q -z?q$a}AE@KjI=+PfIvuSMLr~WC(}i0t(x6)kEpn!E+YpL%PUN{PV}kmW8nP-Rni@PK -zgUA)<5X}7RKxLfx_E-X>SbMZMPKaC5BVyji)atkdB*Z@^szE8f=(opDo>)~6`D0GM -z{4p=}owm?UJjBFe;J{%8XP0Jsiy|^Ba6?{}r-zVSA -zsdwf=uufF(fi@P3*^uk~erlU=#~a%9ziict_pV5xdc*xxbgqwNG>7xp-)-3soXq@d -zfg8OAH21*w?;Qhj%pb5A67ZC_GDa!3CFzNJphZ-~B|bbyotC>KwO;+W*W%Hhb4SxA -z$1lsV;|qFfPMXHolRJz|NA|Iq)Ce5ym^YkKLY*kfVX`tWL#&58#e(<>XJ(Z%;*;4d -zN_@bHB1=&{?=m1;!Kd*gT6a;{He4|Fk_P4H%#-S<$gWwsO~h?fEKS#habg^&j)jO% -z%SzJG_$3kb>bbJ`aPB|xrP3qLSVn2Pf;6VJXgPdjtJTId*r0nZ;1@Nk5) -zjVGPCtM)h#=4J#mm&>3av*+P{5EFzyBMKHV>CSY5y?@T>KVjI0LI_ShTTF6f% -znE5T@=1TX_Qx_*+tClTA_On)4n*3TuGW+NT#ZHe#XeO~xJu(4NQ}oU -z{LlB}Lc8iHB{f-U`mT|^ZCY>t0u5~qXR?U*rGARAUzrAa+Zl6+^_{6Y&~#qCahqBvC`z>3$m8F{EA;RKKO8UxY{8 -zzbvLBXpGP;MpF7D+Uax2L7#a!7L=0=W^0K{pw)42SSlgzlXy0(L&|nVaM_kh=-*%+ -z#&u5p(BFJbOhps_6TmHBaqA>u-;xqM=IME!Tr+_NaxQ-a4%UZIa*o<-x|TK -zu~hD+fgGVFqN9jgZ8240pfW8HzytN;+{7W2W!WPfIHO(P)|sBKgE)G&zg>uXMx-wD -z)j<2T-f3D*h&Fjej`=T@)wReBTP8Q$p;qAX^JrM)M&s(UVRQiZUY%)G9j>dLjUds~ -zDu#Q+!kmq;{BGmsSSU%^uOi4SB*4v59e*Yw?-v`FV(;9h{v4mw35^jcmu*~<9f`&W -z5`-Ltd&t6`j(`C*H^H9cRDX`K7VKAG`p6|%#BFdx1lful6S?hmtTiI9%^{y?y@zTw -zf!gG$+SIw)co=SAtiY6n(^`ajk!uX2hnOA$_8eO$)QK_T)G*v4)Y?Os*|{JKV-Ilw -zcKpm@8mAJGVo!)6k!QG`I(ikCsfoZt2;bJxd$;VD5qJpULY>IY|BfgSUuP2sEci)i -zfP}z$N>p8P!(-uGvpx6mh+rk;o7vpBI`$Zd_cSHmoqGOgM4pGlFeRL`iY4<$&}hI- -z#ga>@RghExyk8*zB>m10{B$sUyITt_`?vXhAt7JoD! -zoyoYCdf_t%S;RO>^snmqPY1_gFmtEs*^0q&6wXr5>>m^d-aJuHsmJmIyoorzp3Vl$ -zo`EbJDi@c|~*r2^y{O)knG*MvI#) -z)TNG-WeRhrc*2SzX$ysjUxpdaG2d^hH@%BGg&h&5V*G;5L%5uJ{+){QOTv -zx*D|+&zCwL7)D&t@*4c-*4LZzkS9l1!*8RlKwaYf^0oECyMt&F`T{K{o2`x4)MVw- -zxiKfbCTU}B;%GXl#i!GYUHM&6?tAagylW -zXdIs6+e48rWhwGS43M|eGc|>q03STzg!NrBPn#U-gVf}uE%+ZF -z)M7%ImVq!4qikI)eFk$t~D-oQg-lRq(|M~#mjPj#( -zW?CR3B!|^phe!i|&r@g{lb2HT;=?FyMV|AiH#yOoUXI|r>P`2|;PF=m%0U&R-sDaT -zUmL+G>P>Pw<8K(8?t=U@JtKm1Mt<-ivu<$uy*hD0;$Mt_2q1nk|Gb~>OzxQes7oyM -zI=b)TJ}=WHGqYMu1a9KQ_-h@TF^F&I1jk*gW7Z7BEs^YUZX0cX4#6jps35d|tQwCQ -z$Yr}%_IGt_zOTbe7vr2ZzO>$T%jmHufq>Ze$AFC?6+<~k1u$y -zr^G7{)0gaGsaLrZ*H)MBgxB%*<>%x5MiYIDUIXC@c(JE!-Oi-Z^woW|51RS(fi|&L -znodk7@2CG<%P+p4Zn3A=+)w|ZmjB27^g4U`k^AZ2)Y^9mB5$x}dwS*l^tM`l(f#~2 -z_ViEgr=O|ipBPBzKCUxuG7GE32s;my)`<&e+5n7DZ{yyqGZmZh!snt0(>DCZ&}ks6 -zj(IGCC61^~c5Zp%l$~J2Nj$2n}nv2qeM=@4}KcYnq -zfW2L79QmgY$`3&Vb2~rne*P78rbL>57R~?L;QUKPoP;R@u)zHv_WY9v;buT_!xX56XOJFBv@t0VX5)01E10aJMG1v2Km^zUl!IJ(UJCp|B8wd`e -z!Qz3S2Mrbu1PKkk)gR=Q2!i@s>=C>IK}tiBA0}$!r)!hvYg7NBO<(EYhwNH9!+;-L -zX*sXenx>r)`9TbBtfg`a@RifVb^VnvE;RU(BaxNS(2I_8SZ5k~uGaK`SPm~m5coXn -zz{d$eiVsm`{;u_&rPaKcgDs8Dy~un=HA7j7P0mV>O7m^i)}dzepQ`;{hMYOv+**3@ -z#yCgNABj(*q>Y<w8Wzx4+hG5G61P@t(l4+Oub -z!Jh_#-_c-ie~|4#5Ii8*BiKs_{(pJ$f|;&H-Ol_sw46?}si1~j0E>3h;CVBZcM!Tj -zL(Tn_2w&6S8Al=`hE6-mVY+CV#cbLrmcw=;2z<;Ad^!-k&y#bsnzfWC^F^L)5P4ED -zg!1GpPs)=8-iDlc+$Uyn@)bI@;_v^$mAlPCGp)AN4#~9tE#!7HbCyDGw?p!+e+#+I -z%r{ZULPzcA{uUCl%qa?)XNP3J`di4?%&e6{zUrvG@oyp5n)!MP`H~%yIrX=Y&zYH8 -z3c1Wt`^mqBeA3L;0MahsF9tZ4AHELY_$6=QNtP|1)aU3zGfvKhYCvM{z;gHa|0C|r -z1FEXlKH$Tsz_tJt6%Y=Gv$t~rT9#TD$gIo;NK0)JVv|FIS=}Z|IGCDIAy$sJSO+!& -zg0MiD=gP{klqMUznYFuCR%%XZ$@hEKKIgFM_P+PNe|+Ef&)$1I>zUTG)_T^k*U&{( -zL~9YGly`uo2$hNL6}D9QfPnN^`UIcCdQqXkOnQX7=kz&+?#%a8N|c7Wy)#&>DlBS#vcqz+Qj -z1C?b16o;=;gMHLnelw_JyVMZKm)s4l&dsL6N!rR*uR_DhP?@9UM$L7v2dyVN&5 -z@{lg6i&}n>F-Cj*S2|p7aM2dJxmdbnkwJzXe(xk6dYhT=nNGuAMFMExoiJdU(+&Gy -zb=K0{()3Qb(aig6LtecH_P^+?-s$G@ai_k6C=6Zp2_9OlhX-u2r<3QkOTTLy1Q2*z -z(v{G7#TK2J5YBdWa-3Px4gd2hqFGlb{a=B@d4)plo#J`|1YbrFUZlBm8rn#iLWaEh -zPW~u(>ji*_a=m4uGB`RjHk@%6t+%|ESXCN~z14i;~3Ol9uB-($KXHaetob=}rf(}Zkq}WC&2@`QhEzJk<)OPe(Uhb4%QADTd2586f -zJE5`&PHM-^MR}d_IuT@RI{C_WbWsaLkZh==N=o`DW!Xdp8ybk;mBcTnQ(j{*xSaSs -z-6=n31g*sHiB5U7!SAt7zKr;->f=ZH;Uv^JCE4iH(Ja;jo$~VnL(Cv8b~6{_-FrJ# -zgE!0Q6mw7pYtoI~0t>-cs5yqu$ZCMc=zzz#p%WfMd(v&UKifnUS(e(V=Nd{!wPX76 -z#!lTKf}%-eGduMh5oFhPia91dTLcZ8_$%o`dojC}*eNeJD5)3DuIN;sF;EHDkac;d -z{B$3TA8CWpF6q>t>VqM_2Y2!#Fs}TdTW@}tY<)qe`fxwIK%*pS_qp+mJ2U(`^`-sr -zf($%)u^aC%>tV%MCD|(4G3M6^o`&mEN5lCn6|T$DDQAKM(()JeuVReia8aM`&=oZ(OjD5o2agwyK)#h2bazmr1Br7GLd}1tL)c86ji4wKRIU{z!P{M~ -zfz8@GwA)0M@F=ii%^ljUB50VqzC)X1WEf9-!GA@$Q*TrFF)DvAb?AoH3_UTVN}r=JxJZ1#7NiDS>9GaNH9o;s -z#ALgqq$=!b<5M=I2;caZ<*aup}7_q@9uy{|*Qf-ElRQBD?hP@4ohx?z7& -zMz?%MIPC5Y{YrQFJbT{NPR{F)FZ0l{IVg4N4W95{>-V|fHmgIw+^yb8H@9m#)QLp> -z*LB6+#!u^zFLjaX$2=*U+@W3UqD8t!c8E%?UL=$;>O@Qj|7R=yw@xs@B0Ko!ttcDg -zOt4`c!s%#Z-7qC}n38^nvh31KM?8z};Cn!U25vOsS%i^cU@8iwsrC$|+M_Cj6lx&p -zV-2LNM${hoS*krj9kQKJ0$dqCT0s?@ko84%lc--` -zaer&q?V{AU^Y%nL`FlA^pb48I$bM;8Z7wtyzaQGkS7;F)Jdxgb*BJNPcK$aMW2AQk -z5ZJ+XqaN$SDRiJ+lzuem!w^KZ-rp|Fpoa-&&`q>|in1(0aTx6t%02s_T@DvI@X=I5 -z-fNdfh@erg-)@(Oiy(W;V6P5yDGYv8&*cMNO6nLTJxQn-88+XKHMjFHu+qX6K7?4- -zx9bK=YIDPW@MX0^Z*90y&#T=ca9|m$@aSyFFKw5@4EZJP+yZVoxSof~+Bo -zYLhkCmUb`}6?UV+OB<%J^&Ykqu5`@X;<-f#zu)T(%!Igg}-203| -zrJnn1+>CZ=8`R$jHLabRZS4$Uy35)fqO7T>&9K`aTrgr`f0NSqt5ot2;u50JNwot*7j~jSbxAR{BkdO7p`OE6o*6oVjcYf?$l4DE+Gr8hzefnV8926fYI4z<9X4>HMOgcn -z3nx_x6@{IU*ob!ZN12L%2!*w)KZp=`VTAKXZ^e|gp`fNThj*+Ap=C4p9X0QtySyzO -zyF;qGDcCjdOLrN{bMqFzFTL9}U2qej-Zt&qKl47%I>rhjGd -zrqJOw{hxp4{cW3cz2JSYKktKo!TW%>aQpk)^h1J;QFe_%+lOu1*KXbiybag~ZR){3 -zn4sO=CL0PFw0s0r$z5&w0XH72Y~xS0VAA(23Ij{Y9cn$bVt=^Xtto&}?S!P7DN1Ql-N5!=|tQ>fH^EVEbJ_&vn_BblvpvHwtJ -zYum&yL+_EIj|q+^1jo~&{Gk49X`_;$y@|r&sXyxt#nd+h5ORUH -zPT%b_XVTsX{7YRr -zJ$Qlfzfyh0mACqP$_;@rBUF@Qr>+m3#HO}wEeN8>X7zbLs;F@fT5!e}AXowUm5gO% -z6nRd24v}%L>-z|epxfltg2ZwuBmUYy{H)c}^P1SGw!&>z?Tk1vV&Adm7<=2^FX(S; -zPkL_WkrwRTz;{i0F0MWqJ^-JGY%9J`vV{zUO$8r$qo1{d_zY`n^}$kPeLAd^kTP&z -zXxrf=!oPn7o?Z9n@{P^aErQwoV5Scq;Id8R%$$HWYuxnei6Pk&aiptuwl{JO@Ii4IDs-%_J|Q2666#L^ -zAOGjcUTgeb2((sb#1-1}%R}-G!)TocT3fM@z^Yw>Buf{5?QA}tf|EP2E&Nc$a%_RY -zu;oxycI1=v`GvLB_cJeC6B9YJCd48xvqL(-SANzn?+4!9ieK>6ruJ~3T{UR!bM!oH56;4K5DGH_mYiXsLramh`ga_X*ThRE`0~t%8^jx{pSG!g| -z5Xg44B4--HaTRuRaHc#UGt5_<`3YcSIQklutvF~s+ruXkY*|^W{H!cm2vJ{Bv~IGQe`Eu=G&Uz%zt)Ji~D)N!m|nJsG*3t6RHL1~w38O~wl87Q7QOp2agg -zW2LC?inr3;!?IhcI>Q@0BBPySSw?WA`moHNXvO*jvJ?9|eF-DNUIyI<%?6#mRM5R= -z1V`!*%IuwH97^mUy0_>nml7%HqPh#1+KMwzbL3rKxYQn>+H#pKXdNFUFDLvv=}TA? -z_(`{u;ZlBVO0Z|&Y^)v0$i||F;5i}g6`ZsRMA$UxX2Lbrdgjnt&rh5Bubm|8gJ=9D -zgqPFSWQ)ztc2H$=q9+pJxF=ix($`s-p?&GU_JgMU0&40D?#ZUeAoUAhXVGE(bAJ+s -zvMz1XTYLAt*W^sGSUc>}rUtJ#_cipW7+Qx>&%U-K*;}zG^C4SOa-fru=A-cEh -zEPS*k#9Q{J+(J4#E&A)dTNXPDv*h!3Bz1#Ve*T*i0^VNaZTk$1Xbu%-9k0r^6=ttp -z(j)~sQ@r<8$|uAnb?A`P6yId8fr*onux{UaRW2QoLS;Z2nl`5i7)g6t(v^ICB-__6 -zSm-<8+W=`m_o*km=lQ@{O_t>Dk|(_VfA5{=edhoy5~UY5GJqcK-At5gOqM3OY$eu4 -zH?^oo#;~TA><2C2X`An~=8FnDzc{<^o>*4f!Xu%zo-CsbVA+Mls#^3UIr90!!bdDP -zsm5v5zW1aBWu>fz2Ls=__k=n{w(dR2N?JJ1$CuR0J+gUF6XxJpi*|)8^P!`$e{V~R -zI;{@|z1O!GC6D~Es5b`Qx)%NNet0h$c=E*lcxzhJ$$fY+qsWRj+p&AoeXZ8n#^P#!9?#ElkKc( -z$0^TP_g08Z5KH3OEiL5X4tP5jWotG$Gt^tx&~Y^U4ZESAK32Cr(vbLTgp}CqVTXoG -zYqSCKh3D;tDQyO<;6e~;mDS)kr?(p47Oo<<|- -z8m1Mz*}IL@Wz_?PDwNJGSU+(YZh**4h-RmoQ5+(qL2+N?>H~qnPM81#2ymauRgrGWj?eOBL0nL^JZToFTtbD -zrQdIM^jlh*`2fn=Gj`lb+0@K^5ztQC5BRa2&3tDQcK@BSvzlh^4PCJf=cHRMt8{TW -zYv-pM%Dk!f(k%aG=O-G!<8xqB#`h@AYP6 -zt1Wh=^au(%ecpo(hO68yY81PAezi-(aYGrcHWF#(Uo?PGL9=??o;^0!uou$xLNosu -z5!#P#9mRi>xsCcScDH{=8~#>fwJl_9?C4OE6>4$IN@{PcPQ_l7Yli@yFxvxHk_Rg3 -zeu~38vtXb!lXWI|5`I`SfEdteoU=N7^T=IDhbB#t#=U1 -zu56aS6gu!XDR_Ca{Dl$hq~K-E^5-JRE@|d1fT*9j6gJWoFVdy;2obKdj&BzKuJ)N> -zqnc^7E$=nMY|UJ4Ky`TEtsiubYUX!R{_e5E7?I6<9s+tdAsF(8Hyi6o-*eL#(||*o -z^$+^=;Mh}cD9Kp -zka}HI$IdkIix7|z@MWi(_&5aAPCGl=gbwScAcGFZ=upgV$H|)+-%!887F?r-!ZMNsFA{FlP)z-j=!ES1^P{y#!fhHw&SZCVs3Qes-j02&Oc) -zHSr??rZ$@R)iu==9i#~WH;j*hNVcO%-O-nCOl!jQ4D8WYmnxcQET&a>WFpT>o5=60 -z+srU0DDihxE;rG>CTyaYn(;O^k;BlodSL9;Cf;G_yWLI83YvJcfXS6^7!_%@rb#Vx -zsp~3_&o)tn=JfCr6NkA?dWi{-mx38v-lS=LX*JrFr<&B_J{Wr(8p}I8ypYyMkgnx0 -zz!tY2fGuqj6Zz_96KqM7=x=WF(4sR8r+jad`dS}<(55$u2yMNI_Rgkh=&P!)dT7ZZ -zv$;*$CQ}+fdrOn>)ayKWfZc?Ag&?iK%}>n!&1}+N?N58P$VWNfOq+zX(Qx`jH!b*G -zVZgK(+%PspSQz^q^Xyo@{n`1xvN2KQrM||@^CD3;wC6mq%!0tRW1I5V1r3Sqt>m&CTGY6S~bYrK`1A?j8$;KLn(yuMI^Fj5Zvv{mgd&bUh -zKpnH#Laqq3jK?sMg5nq>^mjnc01i*IX8X={}Hul16=pY~0Da|mmw -z=Ph;8m+FJ|gRg|V5O}aCBy%{O3EB{VM`_>;OwrR@K*nW`n=@`A&c5BWY3!Jd1FnDU -zn)Pj^Uh!p(&fTM@TzasU3P8lfhM-2@vE#0vzV+oJ5^gbuA;{*h)pIB0=nG-XNcLJI -z=XEH<_u6?C-nfi$bEAAO>^TD$t!-%JWdO>VcD@yFbg9=hs;d2pKud1O8k>=3wx*uH -zNc`>rzX+DsNSUIi+xcdL-E)m{I@tMx-O5J(DzQtm6ArlLG^z_s+(em(f2Og2hmgwI -zlawb}-=*R#pE%zIb3FT7sk23Cdf3&kmT$GQrH$m#P~YZI6MSEz@Rzea^$D*+wghf&1HTEW*KQQZq-(07 -zD~4+*M5qs0pJ00q-bYo`4r^8&J`ak|y>mjc{%)!;s8vLt?5218AL&u8Kri}b361*9 -z{&nIa)G+utdAeIqHm;Fdsk~il$A-Hx!X(-?_B_vqJMbd6%3>Sk8TRML!qP^a9a_&r -zDZi)NX9c0E-p4}f!SyI6XC2ne%PwSt>Zf7Jlb&qvev)ot6`Nm`C#9O*XQX)uRMR4S -z+DHWc{MI_Opj7o$VjGk -z9w)W=(9Pt!V(%7~Qcq)!L-ws$CEFrM=2O%=d}jBp+UEaA|dKgT?2mybn+-W-v-&JqBRY&VlT`-v6eT$ -z5$D^2#S}crsA%9$;9%ZHkFs0$&S6^{cq54Qksf|Hg3FqJBBiJyF8hQ-jVe$htykqF -zCZ0%EM+!Yre;(M$*Mf@{Zf8q&lFg-&`Ut@QG7#@qM+I6N?%~OAgAA76phvp%Q{d$_ -zi2h@k2hUh%wYotK?}wM$V6^vxO){RT<2!bu{}^h5JyFN6AWMXpV2?D2-+cOoCfL#j -z@k36#zzt*fHHeyq`Q|}(xcWc(_I_tJ@OvAiJUz&bi}a;6i2hfg08t(Qc1HvGEH%Ib -z6Jx}A4cZ_$OX+%cD-x=eWlOL4UVb{ -z6PHfva5u9#@9vCSIw{)|Y^k#@os{Sf8YEedEuBM?G@v%>2y7H$$kI-@Wr;q+pa%U+ -zY~Oercmo?KLOw3gC?9w{`4`^?^^^;dVB-m9X^>C$!9aVqo-2q@e|F0N>{LCE0ZjhM -z4dWw;_ISN|tdEutC)m&R^3gsRXulU8R#Lxr(*mZ~^S5_^-(eH%>w4Y=nEss`Cf0`> -zsF#0qv7>arXPufE5VODDbr^yBzfw{g^~)UgY&_9cvE4hY`M$v$F*O*B1N8xv${iWm -z+B`psr(ZkZmGlnLq*t|z{$6%rzy6-94+LG?>qT!)J>b?0ux9jb>hWj$OA`z|o`38> -z`TE=h+fgs-`)2~=st?umqUfpn-FTvntEkt$G^JD2kahC08Ma2isMQ`=w}aUL)7}7tS@EUzvsq+49}feom4X1W*ob=md9Cz2 -zI+zu)5E5oE+!l-RNvwN(k}mp~;w+bJ`oH&8ZU6B}U;P?jkZ<+qljK`JGr8|1MWY|x -zTleq%=yP@dhL?2u-(m5`|D6_U@7}?GsFjZBue;M#>Am*;9q+j<>iR(BO!svClF -z2#@u7no@KuT8%m;{08vuAw0gL7I>YNwSUI@#K7ZQYq7ZdqdNKZzH%wX@_Xy_f0*(; -zX%ca}*1jcF2_2%?!W>#Y9fR{e1Z`U_pHYkc^4eHef4QkvUK@LKQ%HV+ -zwR;S?ays6P&9_QuZIOq(yZsRmdwaU0K|8NQaxWMAoij32RSPkKTcgo{PlMj=4WjRi?4 -zd!?3-go>(z3THvBsG&JXENyq)b=sIm5cTi*TJ>2|J^=gvIv!Jtmj4;IZ2((T$0Gq# -zpEkkNIz9|AIm-k~tK*@7=})?0LdKnS^78&N=88U``nW(Nn?lA7bu>{UKjwjnd7EqN -z)Td0eTd;oZV#OAfnJ|J~TSp}c3;Z4d37g>#JxrmiTp@KCLg?!xim>{SU=DeIKF<-1 -z*d=w^BSMrYpz(G3!+oK+I(?Z4u?d2cY=lJF*|JOty+pCuG80Djym6K(bw+=5F=SCV -z))ijl3R_*_g|6^OSNLAHZsz&*!F8gc#QeI7Xcp`a>KPQgz!SU&K_oNK0O;v%UU^+& -zejWekaeM1Ztd+1B(&Wt8B^Us*b328yvKougiYJs2i!#Da8FA`8u`^E4Kjc5n)eFVg -zk)3LqFpLrW*G_eTDfrz^^=`v#-|Q3!+Fc?j?Dl7VG;7Dc*s0!OFc880JJs7w!H;*U -z^9_C<8Omw%`uLgcw%dqC;YcMzz2y$7DgA>t8+L1VCG{q^-B{~R9)fO#c1x_}zzFTA -zEd;mO8TB7!wQeVWjq1ZJH-1v>PIwE4y&6xOozr(lb%K*7coH(W)icITD*tcr(kyAt -zofwOqN(fAPeP?2F;kCXrz@&dfUE}C4<)NzOQZ$`R_|>|D-NE?>La-pBhgs-3j+V=G -zEmGpS`{q9uoc$Pe2bX0EaAv3QfoYbUm84tEXcB;*%3=i( -zRxibM_-1`^6e};vgZVQ%u`&vb4Juy7x-E;Y4&Y_>tI?DmGLauI}HQtoZEMH -zrV)|t4zg&$9o`$g5|X@i`R6Hmp<}^PUrh4hd)dNV(OyZXrJd5N -zIVn4p0vzge(QTpQl!VdBGFfrh?hTDroRLb+2{rMDooEgUWR;+?z@S8FK+ -zG4MIcvIxa7{NB)^iZk@F&e$%vi*Tax^Nl`*&#iloT13>RG)wm(Q18=$YQniEytGS3JssFx!v+btx;+-{i7O!6V_U+y-u`3S^OM(0tG&7(@18#` -zV0`&5iv@z$bZpMWS0qkUD)XgNKQqIQ6L=S-?h@!LE5P}_j_U_{ -zl>!{^Yr|xE=tw0YTv;|uaSV|*uE$1Gr=-*bNS|*F^~GJ}1%vn9!N1-`3Fx^iyR(%( -zu8YL!QW5y&acn@=L$khzVBVEFLf@Tbsi30hgbc`|GMu(-2Vb^> -z{N95*{=e~i(DPP9PvLank4{$q@BS!fCgK0DeiFUe|LiB#nedbPb^ZP1S@ku;Pv%%s -z;z*8DezA<4oGl4wIK8wnBq7OnHH97Oi`SC5lhv2V0l?mls*aRcB3UL=Maq6G*t%f$ -z(oE|zd2L+cYy_8z1vyW2|2MsUA(H=-UKk_8WAmf6O2Gnb?Yj-fkR(`@Qc1v+ZUZ)@+-$Q1d|X(3RbX@A>9FO}ce*3a -zn0LaWZ%2AAR%+)guVu??`4fcNW25EvUAA_q?A?$8;X;nvI(~}*F!IJjDZX8{&ixeV -z5r?mJrFTj<6%=L{BuU4U1|5z^#G&-I{Zs>b(%bPgBxiLWhwP2P1V|@Uj;H5-ogtJ> -z-?{G|A$R_f&NTYyn*a0K1@uuc|LC#1>ErnP<6qxJABX0j*mnng^vplic{_b{%|Ekt -zK7AC-KZ|F=T;JoulOCVzkKmzkZ+_44?kV=`Ng3iC_trZU>RR$9p3Wt2NoVB?Ea-=% -zS(3&Q4~Ko_b6$AcQqTcfHJ3h9Ggji6kg*D-i14fi9&TS0csN+~xw-V2nz35oJ?Fw3 -z`d{)3`fvD&@*GpkSrx{AuZaSEDYGOEBHI7#?@<>9*YX0;>YEH&_}24Oyi@J}eH)Nu -znOP8b^Q6Q}Le|&Vo!c!**;Nl&YIDxl6y6eSjay-?qQ0CqEhNu%MA$}c{LZL%NtU(0 -z@R$=d>|{;Oi5kg=pCHKzOQ;Ye%qQa(%r6vACqYuF^-4(X5>Lvegw$Q)DVn3y-Qv4j -zJl_}3z2ey;o*#)Pbw(0WsWXz0N}Z8}RO*Z*q*7-jA@!hm9uiOLj3lIfC!UAJlR6^_ -zsni)sNTtq5!othM^K$W=f@h_V6fsRoSU6SxxtD10f*!w*L8*lgndXKu)cFfpW?*Z-aPl5~}?FkUNIK_2d7+QqoK)Cj)5CBj0ZDV$n6Yj^aKWi>fiLgO=yn!fXRrcugc`d!(o3xi&W3 -z^{w7!V_`Kk?IS&{-D+b)T;KW}8w+)P%lN*qM*LPBq|UanL2mq8Y%I_P*Hdk5U=0?z -zsWvtMZ!N>NE5O3M5iMufm=}TWu`vsw;C+r`qe!}qoyA*Dx9QaiF4n8YZdn?-R-Zs= -zm;KyhSNOQ^IsK#4W(gNZ{Vt{hqTXd=?Qne<2n@1ff$vP04c1hNK^c~hTqg(!{~Ipm -zLU7vk%^ui4Ilpl`_RNixZ*s#R>oZOhX!_LxM6v+;h>QP=)6K9Cz>)smUFD_~0@!<8 -zyT)ex6ay_y0QBW@q2@F85yVj&4vl2PR?Tx(61y$pdnGLmCIN4r)d;=sng6l -z?1cHS+-<5E#<^Gqm#|BVj)Am)C?;$>@mpG&S8+3k_^A{1JP7uVf3(u>=t4-J?9OoM7!-)cRWbs`5S&YPGZ1qz1m^Ro9ixMz3)B|G?0)p#kLOd|Oq!P9apjvdXQjB>G2b_0M$LvBL~9) -zN)D=ffCmQJ->L~J5A?u9nom`$QvWm`6L^}R2M@GA37G2ZfeG5f)!IN4ZTvwRtJrP{ -zrw$Tyu!SE6Esf -zU1{F2e%*H2BGANwo8;|kZw&b$5lY$)o)ltNZYO~DhY`Bm2>mWX(CaUFlhTjZ@M3>&JoUj-43>qLI9tELG3h%gJmjQaCpwS3HlXy83pZTRCK-1dV_S5))& -zD{y|u_ik8HPepbD?hOx4I#nqOR)wohJXKAmL3#PkBZs$xl>NP0`=L(`9|%v)9;#Nq -z?}M=iKr8cJN^L0&oSZ; -z^u^DvCKn3&FFY{eQ*W%651Qmqc@im7Kl9*;!Z5vBJJ28RZ$gIti3d;UJ*8UyqCeiH -zurHRbs2>SL!{!Oq^xsDQ&_fHk6NFrSpBXl$n%`D|^?iFyv~smk-ajzG1{?MBeGjdu -z13}elPk&v88TIo$4<6(iwCZjTOwe9ft-ar$Hc-&&Z+q}~as~3#zgq5gAtG(@K1Myx -zch$xCUQE)UUaxV7&l%y>t}x`Et|EbImxnK4zgAI?OYStoeyK9H54Q`TVaFd0nAT>3 -z{i}*cLNT?~1pB6n4+BhYF~Pp7;-P@)O(xhELKm%3fL!(HQ=yC6;DN<|WTfFpUmEsS -zQRd5aeK_wKIEVXi-Vwa@S`QA=`9_s+EH!2rdS+w~ZiY2iiS;4X9+=3Z`YLrt|2(QS -z^sn^bK}MCqufhx~7joroW|#(BQu|OQfQEfpl_t@(fRpR%c{tpxF -zV=n!-m|zbJTdJGQv=18mHko0ItHeG5eS?X1qG8|l9z2l`X;u2>{`qi+VUJh(V}TkXLEzwt(WSY?KdHqyJ&0~0>eUZt)v -zr3189VS6ptg9lioksl5-Y`DQM#|#@HVES@1ELiBGJ!6JRqAsdWn_&a1_?=Y$vjos6 -z+r9?BC(U@4D(*$|Nl%#Y{@5<|kf@KFV7~!I%Ml+l!A@?cdZ9mR=67^ES1Yl<;}H|y -zk?mBM)rZY^|Jp9fn!L<}_l>~QmzweZY2axOn(#i~F8sv3w$Gr|mzZex -z2%WXX9z0Q=-`%c1&_Cba5;Ek4{qZ`ttM~QCYu!$MP*Xj4$lpf8R%(Vx?oNZ>J!ZUW -z(XGyP)8k-w=PJlN_np|h4^hAk6zQExWG -z9x(FvCNpf2D}QegK%|t@ -zzbia2?0NFCu#2|S+yK_UO{31?*zIBfKUE+ZcEY4W1gi5C4;_22f)6TJJ+@>a+ju~^ -zIxB`n0u{++;Yf~py_*NSa2ryh&WvG$wxN5bUq=z(%>o46V=svBE$64pus>*`n^u&M -z0o%37F$Gv*HC*z}AB}|oMqikpEXTcezn1HlxC{)QF@I1Oe6;s0pEw&W#}{&bE64o_ -zi81VSB|liE;^R~$6w?0|!;ZURj#l!|Ma(Z0qao(Ya{h6djE~RD`G;k)uo~(7O(j_a -z>ruwUV4CHtN^<7vXpe0G+f&YWAr0~FmsL1+>B7<3dr5;HfC|$MXd_$R>5W{LK`Kx7Gyc@o~ -zoEMa7aYE3gIhLdfvTx1{Wd-(-jUlNa-qr=*V!cOdiZ^~V;_3Qx9OBL+Yzj(xU5tt{ -zM0U^}au01LK>DbVMIp%{-YHL(CB}V=J7j`VZ@y|>a*Frr5X3s}1qvUUk}l*tNOG1|>KF9M0p5~Iz6cTWK*L7R8DuQ8Qk6`+ftP0R^XudH -zQ#tx6cUI~Fef;=kC*GIFSiUnybSAvZ>*I4`{7+z$wLvL_FCO# -z|LRKqA;h45pSOZhqEJHp!yOOMG*Oo1-vx-$Dg4&tO4VZ0U%)0-%DvXUwfQWrl7@+} -z*BN3xL4>r^9zJ3WIjT}WXQqj*oQ3I2sB@BNtRkeI@X!DjQAvY-`Bx82Oc8`ts;5nS -zhzSd>luwyp>;ktwzYy;rL!X~Le1txJmHKfrjh9=WBSdq)f*7kmdT4~4GZoq|W|~V; -zk6}VKiQea|?05x#w^TiE#r=&7yxB#7YUhXp)t4pPP@1LjJ@%EKrFEr94QUA0WUO>L -zy`3r21_~8qtPFC{?#Jv~bC1pmNllS1BX9zplw|m?jxwBMa>k<*^lS!1x=%l7hSism -zp|r2eupI)X{?h|v)qrSywq=!N(Dt||UZl6ZLOW(f;}SAsed3q -zRED8neg(g&6!Qb`oA7cfY6{O_gZCa2ER$LxtUvE|!$kdEU8e4}7VQ2V>$5`h`|fiU -zaoJ^H`ai@yUZ!{dPjO4jpb6o`=S&H)!THNxK)QiOg;AACqs*-*M -zmKXa-gIHn(pIwUWS4~tmhLuH9`D(OclX9d5Yb|0WZ-4BMn<#l>T`BNww0JII*HJtW -z8j1T6J@akBWgG>sTi@xH2Yx|i9EC4W+hKzFml^YIwQd;m0|Y}n>qkj)jg@ZswUqIH -zfR)Sz3Z;krUP{A7py*Ws<#Z{UFyv?zP$>RHDGePdzEZ^hQYxg#m9F?7O8IKARJoO} -z#QU#iyT2}_+3vDFnO~Lir$~+MR?4}*r1^wsv=R>u()>XwHE614hII>=TGW;7(^OWHN=hLsYrS>B5Nj$HCnROh2|4Os)|I}!*R%;X$=#+4f1I#Qea*xN -zX{#s|{%5@>9YP-8rf=?#S0&1@{E7z;wB_5VAEmGJz{GrE$u@a|iIzXK6>Z~F<@&4E -zD+0TBv&YMYbI@P0(psRvto>4e6k0eY7v}*CjZ-3Y&G#wpY;fpAe2uLPN~5_ -zCAMfApJ)&`AQpX!ZTw;*oCCSgo~3W&;{Z{!gt=Dg(j05^pBCXfCgHO8BdbI$^Pk7QgAU&KVxh?XTwYyddkGx$Pk1ps1`Znt@ -z#7=SX&>yAPv7Xp6>nOw~Y@;y;1U^EsGu^RrsdX}9$8Doxh1g{jyT%h+V!Z~jv7+F> -zq)QPS8R(8xi>Y(_j4XA>PkuHW0s%;+e%0zXf)SWPg-%zpc^{#AhO&d2jW^Z-OBs -z*~xO=QbMIc%b@ti5>NaFvMllvm;KG+|0;LiVTdtIn#bmdo5l}%;-u&01*SOeEJ5>e -zpghqQQtrg{hw5EcGBNw8TpDQr?zG~{_K&?p{Sb1xJ-je{thqeM5jHotpzD0tZx`0!wyMn+c6FTzn|5JO*U_+_ -zLJxHv40|#3cvnZ*#i5R_?67x23VJ%i8izW2vcsl?*Yxz>`g|}eF6a61x6%YT#mY*{ -z`9f%*&9O=m_Xk3iE#>-bD;3`ffs0bu#&UVKRW$BT2aZW%8_MNdtYl@NjzorOE38sb -z=iHJBfzluf=2{z4-bp$z;3xkwuRpw#yNS$12PcM9`tUdx|72^Wf2*H-sMp~3 -zEw1~_NGi;9^U%H#JoImS!zP|5CaRf9w82EbLNHOk?PG#*k}G}Ftn5a2`ledLKKzXm -zIJ=w&8?^EiD@!fsl99m6t?Vjy0xxqXkbVZbxT%w^Y^sZ!cBwV&*Av9;s&f9jW|->| -zD@!crzaezMPn%?AaV~E9#6E7KZNI4e<_B@3#8Sm}zmmLIaop=bS=oSlr4EK&9#Dg_ -z^2NX|l$ET3Jt!+r2OUCLx#+^o<4ak%OBL-RD;s90GDx3b#TMy^a{f64%Lw?e5#{_- -z1k~|XHprc#aaL^ADzp{aY?MFV>f2agDFtb`e$yA%Zw~U*2L7(5kH(?6Sbq6V@6E1* -z7OnQAK2mwNAXVQpNG*M&O`0qVQpC+T+RJ_^Gx9-;#c?X4H;eyqHd6Q3GG0aMB7pnV -z%lI}@H^#~imAOio)#`t+mmPF7h_MnS4J5ovQD>4Ta!45VbGVk40w)etLV -zu0(1$^V5fIE#oTErC(@en_cOW@m*fV?*>>K3c%dH|_%}c0_?75nONGut=h)@wY$lew?y?=H -zY5rhFk=7kah1*2@BYp9)n&vCwc?<-z2g)=b5oGt5@kl^)ZzH(4j1Q-vmk1ht^`!c( -z=nvX5PP*2Ld(KCkRfa<-cUiH6(R)U;MM-{AaXdy#dia#DlQ46swf3gRs3TR -zRmEpQkHc+$Fpyp9GMj!fnoTs!Hc~zr9X5Et&2S8tmGM)>Od|U+QP&ca_rXN{r5_XK?Mjn&IGXi}&MzkL{uK?^Nog8T#ydrtwC`|aWT{bC -zbv>H-m+@xe_Cp^xq=_9Z6%Jse_DwYV$<6IhG+ZZfJ6Xyr4Q^jYGu_SYU^K=yd@CX} -zy&pF*ZxElZQC`O8(1p?tS|8!KCuh8GzoVka$0q$;On7!j);DnpgqD!u5Ql^!E@qF2 -zvmLj_xb`QBvGtdwWRWBK{%F@61>cPz`?OR(5N%8lQUAr8eTb13I#KE;Zal!=FXfAh -zr7z`=O|W-M=|6-1p#X`#5B~oeqcp9@gx6Ne=Yv+=XM#1B8e1Rtx?!xYl&i(6?hcRl -zztekdwB{&WinK+>X8q_K^P&`Ky<;7A!WGW3POeG(y1dldlmEkk!auHN3`RWBW#6MWyW -zoQ*V`N?r6KJx4zr84KoXUn#XF=Wm)`=<8xWW0+-Sm<1byR)z>jpk;=TI|IRBi~m52 -zKQ7`8q>o)trnlEw)mdnqp0V)Zkw?d`yl*{*c9?EmdA3*DwXAB-%5%L1RUb${AT9ZG -zg8fhSMgn>OLvu$}m9ZCZ*Z$t@Io27U_F~h{G~dKIRxsH0VecKj%c{NtHS#KD2`L`8 -zz!zo3y3{vWez+PZcuc3QCZNHg`GtQE!FPUU$g-;CrZe=RkSJ@&*CAWQ_^)1=-Ia$+ -zG|~N1>)d&v)J&K5uL}K}k}yrNT`o1^+26rrTmv7^-VBiNu~Tqk^ELPODyF-6lsXOvXmb#l6OV3D@wUuq#|ls -zDgQ>$O)ceL6CJx86-$4E=o3o$mql_XyiqCtyh!VcX7Qzbf02xjv8DXuBE8j<4l(!n -zWwBc2D%zAboZT@h{DM=qASC?bV#1J1-DzYoqAIHu(RsVYe9_2KZJVH#9PTkR4xbqn -zZmn|!urMHN4W!aw`l^ct$B70ykUJY#tb#lY-cfYL9L0F0@cr>aq!J>jFz=udzIA79q5Fw -z4#YKb)F;1gP>?fhO-N??4f;ptGsCZw!qo%|D-q1m0p1kNiUgyI=ouHH1$anFct8r2 -z<1FkSE);!zG}}Zda(T4=Q8fD}N`zhkt#A*-1Pfbdpk1N863y~SwXM4-%2~dbGbU%%R6x*b*ahg7$ngTS)r- -zBU*bWnpJ?2Mz+;E1*32|#=;i380o8{*_}fACc!9Nj<&FQM&uRx%4l{g7|9!9b@iKG -z?Tu)*k+^LJH_zPI)xf}d+Z7&IS`zu(xijuNgW8*!>pQ`3m#-yj6?QVkS?wLuLlUp1 -zTPzGWK4u0up^T(W5;T|oS2ULzG$=WWD7tD^pKP1p6ZT*633JIFDrl7dibiqK3=}lC -z{#P`&8Z;^AXdfHQkaak2fqj`gv;%vM#csG|RWF%*(omq2O;Ud8k00?r;77Rd*Y2kF -z66bN>ch)$LyTg4PHxArcl&u}I7<#~NW4DTR5Bfr3ILHF5r-&-8c5k08KOp&YGu=lg -z&CC4r$w^NY^*cG~f#N@%oHQ@feQ?seOpolQBC(z-txwLoLXN*jj{l$Kr1h6G``_gF -zd*t|gA)ATg}2^@q4R;FDk;!?u{l`M~ShX -z5(7yF0dwU9d -zCM>5(&q3_T5-N3SB5^V7y27y6-#mPgClfT92-B}H!&Vy1r+HvB8^#< -zJk*K**;53fA>&CydHpgI?9meO??;<#f<0J5Ow~(0GQe+f3F#tV;(}qX@JG&cg)Ml- -zcfsn}3-TY?==YrO6W$9R^j+lRNz>gW{20Ukig-6~Y-jGmzE3G=wfFV5riSc1M@09U -z@=ptX$-7-XJ%GBPG1H1b?y||vCA4NzpWxAh-5?kML&Q^N0dhPK#;Dz>0KX{hZbm=)>@K$3z^4J)ZR*Z-Dd&{LUBJYeXi-~)qurZ`(t#?m~|G|n7y*9Qnr7Y=rUK-R8 -zAveYLbPjKPs4T6fGq^P9ok0x|`$y`Xns(Um^(gHfP5hmHeY -zx9dw!(M!Eby!0ROswrF~FFTbP8klvquqdvuaFYCNbfztc?i|!VKARV=FZSda)lW97 -zNE_6@&RkPO5xT!8FU0g~nogrb@#Pcw6_c)4kpKhPV#_)3_bEjUV#8Q6MM*0g8@xK8o@b|Mk7&yA1q~1r_akwB7Ql -zc{ZMXV103Q&prBYE?&UP(|D^B4(yBv?*)zff)aS@X&0Wj -zkG1P;)~^-e>K$h@!-wi$M`5%4t!MS#Q#j#%O&ktn!jTy0+K$)-e}-2@0L#+IyQ^oS -zw#?N2IE6Dv?l@}+PmFWKNud@S0|lig&GNL@onFcN&Ib=c^+L`BM_&;r=iB)GTBUE- -zdrv!pED_Dt)gjLXUSIz9=A)UhzLmc6cQI6gFs~{FVTpl{O?<#YF;sqZOk%wjgOV1Z -zW+`}0MGjsXLIk!wC__?EPvO(j1@c5M7x{-V;$w^UWy}m;+UFSrf_*L^R=N|qhW^3XiF3|1! -zv``AnsVF@5j)-~Ccv9(bKIjxrlKpim>zZ&9C)41&>;97jUnp=E(oIIrh3(?miD$v+ -zoSLe;s*MfX3x(K)>qP8E@uc0L&V`%hw|(IX_tAQ&AN}AT_s~}d!A_oHzZdgtCT+p| -z_=loMiS&hQ_e3eSU?pK7ZUkm$iq#u&{2`jY$eDUNu%tCJ{t$C&6B%C!Z}Hw{jAeQOe*g;JNX>&!tiQ$ -z)D|Z@P|Q;b^)I9Jad2GfxL9l_#F04Dn$+wd1OJm^KBEu?c2AV3a5$T_r&#TdGT(0! -zKbEYTbiU9QyEf5A_O#uz)>XeU6E0w>h5W`sELZZ;-ax7{Q!ijQ7K$ztAYBAWzJOg< -zC{l&rS)By2(GS4sg`yXsb^t>5LNVG=%wwUA+)f~Z@n{5DRk8X8s-6obTnXD&EO+(6 -zKwE<07wtc6cGHUgu&h{XccHmtY$+D~uLgleFpPxbHxwVVg-o%H#?BjYl%pvaLSE+Fp+-ti&L-fgKP)a7U&nu8-Kp;E -zg9)p!gHCye3Ffq5z2Fy4y`~Sw&NvOP%A-zA35Z!(>^_UrX4sP*b@E&WFIN?HYJNaW -z+P}pgcJe0}da;#J((0|31;otnhav2rb8GS1TZ>nPMy(GW!{!w8ImKH;t@WW{owJMg -z%`SdBbkL`vfn+ev+d}!H`HGirgp54BUW_Pp=CL<9CpM#4E^#G=Y%SKzCl%|(CKz>V -z*|cJ<$ONNqJqs^ZohF!Juc5^LS!YLygPoN7VT4Or#k -zO!dL^Zakw?DYdsp@|T=amUDKHGzf?9?;7cxJs@J@xapZ!ea^-)G0E~$lr+_u>JJt& -zlV6O2-JGd@SjM-^slE{9OubO-DNM7p6eJxi8Z}ALNBrd6{R67IIQwPomqnL(<=Y&N -zB3#GRB@h28uBl_xRCKu#6YfDL<@eq=TgR=T@IpqrJLz>?$t8bHMUnDX^gnJm+dUr* -zP2dbK3ol~q-J+6+0aKf+Ym)S0qhF9_k&}k7H;c5SA#tfm6`06h?d?eN$u4~~JO5-s -zl9w~dlD#^~k>p+K1r%-C5P9m5>unJeC%^IuF55vdq~od~FU`Vs6g32HdEOT8yT^+? -z{aRx;EAX&*Ra=$_(=rx$D_KN*p?!G -z3<3G^sFuJ=uUSFhs$zA~i0;KTF~L$`^`_h_0`eLwy>MRkt1~Um^yQLI-oq@vNS$q{ -zBkDu$sEI-Gc}4xa64es{U-fVBO7By*1ae1Fb!}lo5G5~2>k*f3f#y_dKzvqFoUPF9 -z-3XQt5dZj}U_l9|as%TZAt?SHx*Q>J^Bg5LzT+p3D#E1#k+Y?Nsa?mcb8}-B7hQ9S -zm*iJ{3H+vLOVI6DJ^8Xgu9~z^CmVOhikPGYMTu9r&XMyC%kWcf8KMjhQSKS6#0D!5 -zU#Co*q2ykxTy?eNLn#=H-tPI4cLr{miN5vG-pnC>+E1rZbTaMU+ApWSd=${1dRqfU -zKU$swF`0JiEhpkjiuK1zu8F&TM!u4N4~S3hm)0Chr5C@lC~}te_lO(3Qagw2!GF*O -zU-|3k>!;uL@-bJLjh|eU{D!sFyVM7_ja@(SnsvV-g7{4g%!x0`o>+L9&=aKi0lxSQ -zy;K!1^$|Se2k7ewJX(G9yUySMHo$%p#hiE`Zf*3Jp98i|3?B5n==%e=VE2Df-nW%L -zk$Lj{5jZrH -zC8=gdhXwx51KlVKBX*s^&mbJxaBsv69Eb)TM8E#(*R&g4Zi!%MM-yM82z`dfUq{A{ -znivrOn{$fI=8X=ar+o}_`YEBAifx&aOvf83=~iXgNX0Q)+OP@-B^6j{hCCuNd~4`} -z$Z_?d(L^Vj7u-3t`X>ArgqmIqp#(}zdQiI{%ds|O7eki7)QXRNJ(6;CSR -zGnBD6BJts)l}$1dA8u9Vk5(4T%9FOP-9NKyUdNeFW3j9Y%zd@e5Nak|mCh)-O@;w`TipTds{d5=L}E}N??j8&lc*pbT2aAp25W$_T@$q;1? -z@eL19HWA-&FDa;H+?VrdtsxC+VAFR(sPyY^g{<_m#6RSWo06X}7-hWGM=L`NY5>-( -zdEB$xZu(-RW6N13JV}`!t1PxEPmUz(#VDJi$a*7`vC%PSUn3)_Z7%J#8uv%N`=iSB -zp;n3ySDwhK%SM+-;s3!xV>_>QR(f|d=hsA7_V~(I_GyZBKcu|6z -ztqh4(>{eycNM(AsGJBYkF+_PJL|H*b8WNzq>@PLf4}tN+<_{vpn+e`fr -zEHUf2&#+Fsr7UUd(19dnj~`YU6S{Wu;QIsfo}^DSFV_whALCUev@B)oP|Py`JRX0W -z&yit$vBTUR`gEW>Vj33koDJ!Ild>kk -zjgH+vhrJUBi@&on+G&?to`Fj2mz8o2up+{K!TpAsIA~`wO7=lb?Gf?wc1UrCsy0uR&U?i -zUW+YDi!4DQD4U=;u}2oACA7WbLJQR5QtVZ_xYPx`J&M*^MIi`6zMnY(+rHoT{e6Fb -z-IJ7BwgP+cVzA|lpZ@@(^J&(KXJ5M8i|>E -zdQr~8U`g?Ctd2Lw(q($s;+%Z{ocwxt8j|PCO>JI`)io@lWHC`Cwoe}kpUKEnD9hRA -zpR=tV%0EJnGX`o)4%&GK!#<)Mt@p$!zY*-ZnP4{`(xBq?0A0> -za{YrN>@vVnr<+z$I0}=|hC>HT_&`*oX-b4?uHLjf)U;M(+7M{U^ruZwnTow>6GTjG -zC~$w^gd79Q+r_%;9T#EQYEcENXLa$M6CNKm%Bk6m4Ey`DvN&~m2u*zEPOaB9ywD|n-^g~Jt9gL -z(NP)m|2ZlT=@LI2xJ4gQd&t&#Xi7v+$(hq -z^(2qsrd!pSQCL8eAK{842d={!E;ATDqMu>b`##*?>W9wG!(3x@0W?bNO$ab!wic1X -zxIh|d{0x;2&{0-nNY-(=p#;@QETZwJfFi2mXH6T|bFl -z(}?u+sF6vg>7GZ`tTo`Mu|j&pop3_IzHfL%YI%zmJ|P#f4b5zomvG#l@=&rmFX1`= -z(ax-$+wN4>QF4Qmy@1?vs1{)9p}8(8j=-yhOVxXU_#x -zFat|0+X8r?M5-DY$GbDih1cDD%Wy-*6Jkb+Z;`aH2S3J8@ytLRKT50+F7HkR{J!i8 -zCH{2TebhFv?DCKK)5ZuEmxxozhV>2p`Hf`v3|U&VM3<6bEHoBG%F;$bD@F!gSD%d* -z8+!&@1NXXmzL>Fti5OV_R;)Fw)()*w@H_n#UJ^{082^MI+lo~=D>afAlwvCFu7Y_~ -zoHdO{{F$#@ODYvra>n6T7+rD$yKs_n#6PLqR^{CWA#>}g6{in+)cKwmvvy0Tn*lS#t1CnP0cQ2 -zc8PnJFtE1XaT(J~hG}Is;|e$5;O3i!-7Vif~ -z=;AHb4>`Gvy&y7?Zr(;~cbwSS(7f~1@tx%3E3qCev9iWkX$>xFP)eAruJ&Cnr}jD8 -z9_9;tB))nl2{LIPF)bfyT07jdfuZZQ!BiYZ*Xs~dRB%a@KB`O?wLBFxRuVE@GL)kp -zn*dLrfW)rlc_7?LnPViO?gUX$Kzls^!`cYC3i=o;2YPKBi~c-ZCz*eV=>euFWv+dv -z(Z170Z;CK-;PFM4gU%P)RQ}V57nCM#v}ySxrnMtY8|Y?DJH%8>H)~p*DJrC7Ih|k- -zAQ@fLj~hgydROF#xdX+bjnvvOO?NMihq&X88f=u<23ogJ_5ZVOMn6!upb)Rz<#vg6 -z3#)#woAC^A*A1WIkz$jij?asQ_rhbB?-YIJrVO2&6yrZ^j=W0Bz85|$P^QW3GwiA- -zK8fujH{c#a(=0VD%&pi-QJy&!FdC+S+za$g&z=M}@!!$MeJm?WGJA>V(uILlo^?{@ -zbiga>AD2it)lTNSzBL~6#_FCiKXEzkS1PZ6UAAt)PO^1&pw5&RLOx#_7-(ALX?j^^ -z{(InxA+|TE^aLJM2ZC>lV>m$BoBv*i?WB+jUw#)n9m95>kW=L)*tc}Vpzc5CWAcI8 -z$QSl7pHPe#BLQ~{s=3K;Z#NCV>7*~y;i3)DOZIZ-57!a) -zpMG)PB8FjuQ2u)r2hc6YuRGkE5B5Djx}QbyF!qjX$@o4>6Y*%a)Ovfw5V-RdmPW&KDAYeaFhL{EvDZfr3?5 -z>xrXD^PZVEc6FZU)`#XvzOrNf$`OGX9^(8-kW7x|JlyyCvRZ5|T5kr7U>#n{3YX&uo>8 -z-kh6rd#w7+XL4@8t47|flE!TkHqlV!HRYKpv)J8ztGBi#d|zB1FfTIYc!$`WC1)Uk -z1&>dHqfZo@W+_dpyi6a+O^9LbdT@IxdThd2wjRkslx^_oCLV}Kpd7hWlh71d8erLz;6407I#Q*(&im#ax -z`?np2n4Gy|Z5|SZN)o4(qyn}J^pev$YE;3uO^5$6_mf!x9!Y-pI9eW&}agb=DRT&VeW -zAzUj=pfg|#qn02&Wy!a>V#bs;fvAnn(^ui=@-I91poeuqpEq>} -z&chB3`_uO(?O+3t$8y7NTP1wZ$Bo-THTH~}=0ZKuTd3YLX2|XMM|T`~IV|Ck9b_Ub -z&ssl(Q7kotMdFj*24k!a7d&326{8H~evDWP>-F$kT;vYXpb-b&6=9rV;Dz-JL+&v5 -zD=$ZOly~1ZuGt}Y_ctDKTrk=$y62G55^8t-(uegU&Uc4n&<>n-m4rbbs6>_TgUxd3 -zaqz=uhr$w%28f -z34yh!X-2Fm@=2WSmE$s+y)(dP$(p1RGHt4yq>f7AZk37#a87*MO_YQVuxdC6+~J~m -z^k3;`KyE;Tqh2O<+=46yKP3y(Zo6fvS^-BqE{L^|Y0O-` -z7JW8k+3s;NGIX3GBkvmw;|BT#Tg#g+?wf%=@V8Kwu3S(D-*lM+0=chC9Sx<#GOAa& -zsV6Vs$TBb&ec`@vcifH#9dK%6Y=2n!VsmT?k))7evqztE8@|saOB)qy})-dg3xjz@)v6mW> -zX1$rNzTM-W@lW3D>)K8zA_kw&=!27Bi%`btPnn3*x+hRlXvBKmU(OdKI+zd5x -z#|)|F9EFY@g~Y

Ko=3+fI&~enDOTl|OEzLHemfg+x^oFKc|gx{&7Q?1k2~xqw#u -z{pccEQOuBJ;nuxT-fw>DpOk8_iMNea$#ShIma-<9QTHLwLUSg!rj*XM3v%ImJ-50) -z%zRIAmMx`{bM`xZ?BeanT+`U6YQ#dNj2Y@k`Ph{-*1x_Ym3UWYq$X9WiOM$C0BG5( -zT(+_PNn;K6#$Mu$1?_q5igdMgk1I!v<%_#48?2lpC(mG*oZ8a(0E?#zHlDlu4Tn{m=$CVb!O6a9J=|0X^u(^;FV#QhV@Q!iry~31T2E*`FA$GgJE9xWB-|9N@(cyZ?eR4hZbY -zlsPTS=RHHljhR&s+oj~7(lJliGH14Qt?(^%94~_E`VgJ` -zNOyD>LNDJ|DAfOsx149GV_^}jD!qpT1&^X?g-g^pc#<&KpIpZqD`;S$G2fQ_kmnD3u><-ROo -ztNxVlK#8cd=64U|bL+4_FA*w#&&QoC5>@Jah5k?bF0_X)5$#lY`#^aIrqOwBcL@w^ -zAJP@%<4PPBBrvvz`F50`-6bXLM}Nv!P~s>=0$)0ikA&D8@7T)ogz5nPVAwQeY;H-M -zR5GTxzF?fceODVF*e3+zXqd1+Op`)Uj^aoSNA>^dCH#)x>*oGl;K(Ro_v+#Q#J`XG -zd!Zw(08qr6UmqtFI8qAuN!{kgqG -z$^R%IO2-Y+Zb&p~N^u)Dtx3XcqITF$T=nb)bDh=BI8TW=O39f^pf%jBZ<0#5RR{{a -z{?yI_WNsYYnE2c62h`f8{q;r?-YOEY#@8F&-!cdIaLZ`Xoq!0p6rtZ?2)Hg1F^ta? -z!6oX$mlnY!g)PDIfQxt*lw?S8A2Qm$ZVjILfxik^$^53|X7-e1l=UI+sO -zHc!t@Eu_{7SmEB+Q;SJqx&a`LrP4+f`yUA_4MRHv0+I9>VBh|kz=Qk;%pxbdT+jVh=;+NCQuW-gg>;}0 -zxKW5nCjf436i^g^x9EK1ll9!qqWLNY|2K+8srV%1D8fMv{9i4?0gdpo-pM{96DG)*Ru@}pfDcyE -zt^Wpt`vTuBWfDBN-TAETH=)LZd8|v=u4j`U@Tm`2^F2QC9)_qMm9dRo%%>6fU*sZl -znME$_o+w=M7-Si*_xSEpu}NgM)$wLa!!r*{iMccKG=>LL;PSs -zkg{D*`1PR+8&G(fv`RJy+h?fno)R~?jkaX=89(ez^8{Kke+BYH@)+An|8(qs#EOp@ -z*Kd#h`}S75+QH^CtMrjA{u8aYu^ym5N_I*Y2}`jZ83yw{UvBMo$4mLJuUvo$T<5_) -zh~QRl7nei5FFT(Sp%MzaFMwCVml#CBcaTLRWJoMe@Y#|yi3A5Tn -z4)k!#wgbJ`=k(l??LaR9D7bh#nFmxvWs$zjYY!}yTs*#!#4OV*a4XwWG%PFwG2A|1 -zB`no*Pi`NjVwdW%2jLH4S^>9K>Fbmg0EF;t7z$4j5&0Rh@I)GBmgu?hg^ro|!XiC4 -zmSQ)HKs3c}VWFOj+77@h)B~w-*ljFDbv7Q!Ky~41J@+V3oyGrXpt|r^JvS1l&fjIh}O#54qIFn($m -z3e4!TaygB*qxCohJU<`@$A5oSU$Gs%W*uan30oyfzJyOA00Gb=+;zr$5LxD%=q)91dXb4gsAw5zq`KkAU>Ci??+D8!J2p=2K=qbhCtmFnQaiPU_^mKsd6)uOiC{V@yR?I4U -zxm(4o>p{6xXpUf8!-Wth?f*yUKQHZN-|U6DS*svJiKYq7R-_#7r3$R1zou5ng@2HD -z)c9Ku2KIzkl?SwHiR6T5oeUB`Sd%IEAAJQ636tN&UiIZiO1U43v9e6oBf_lpwA+e> -z^WI^%Re4MH|LO(@yCWj*AP|ln9TrzHaCYw)q$w!G)fWQsC+WHHVn_8h1_6DsFpf@etgS+VYXQ}#ryRep -z=_1TZmt$-33resGujcbcJr!J$u#HL*4&<{(n1-rMQ-r2pgpl4!>YZk{Et@?>x8>wpMo&;oXk)D#qG8X93*r8kWWHO(4g -zUK}Vq0=W}JC_nmQw0uCG1qmbyEnoNOEhk5%!cjn@&DDZsYmnog#l)|ApUu8qQnOu3 -zeS>tfoG&GYfJeEkHh}h-gbO%qsAWO4wah6jL9hz|_Q`3z-{K3xC%)V(7(+rBKfPHi -zM;*D8w8Os})j`;IQI8{HJ?s`xU4Ba!rns=gWuAid3l{{^rF*Be38qM4|B3=3LgZtZ -z5T(@Q_!6cX`!rpw3+RV067KzcCJ{);T!i|dVcfQuBmUSHM1-0mWjF`wm|a}u(fE@r -zWjoC2S|1v>6R30>K>-r6h*E~&EnjoY@P@~W9aE9rD&{Rx7r71LG0w_HZZWe#%)6Lh -zhaNI>02pBl1q|SVc3gF0OOglG0-cnQT}+m8ql&5BsP(Hp@TtBzXHuE_I&N^WNI%JF -zwo3<+F{cZd2a?1n^;|$P8p8YNIR(XjA3fPygQ8mEQ9UJ(^DF=bY(ZHr9g@F5#74?zenL -z?^eMBu5}71;X~kV0SKQ=&)q2i;luxL1t5G<@COATeE9#R0EADXhlI98WqBF2-H#}w -zba5X<*Y%fN>`UktTjk5Vr~@bRB22;rO)rxg0br#@xc3}WghzS-tD8vUlOh%c7I*ylaWF`nGHq9%ItvnO}92$SPVV;^^>i1@Zv5o$CI*gDt_r*pyMlDVwi({7?#F) -ztS4m67&*AA5d=(U@+z|)Ms$!2@&6E+sI%x@qaM}^K&@XMfNVX77k^{PR?-bILM80= -zQ0d18eI>scMmI9y1JUrzjT`)Dgew&R{3|+!)*T2J|Mv9Tx7yL -zDGYCvNp+D+OD>#xU(CG`xg(GlIXC=-;#G1<*mTKPoT`~;`}*@HQv -zQ*VkT-}qlk3acNM)Xh?vm+Z#&lk)wu+Hn;X|xWD5?a@c)*Vyx&-K-LRY9XJtmvC(9+qEA -zwIaJHIic`1F2eu=l>#E@2(p<2`!3^RF`mtU;UY%eAK2&4|5q_y*wm+=%TEH`VE;*Z -z=y&NOh%U`)y27fvK<#Nk#+TNsV+KW#4g5v%tQY3HNBoE){kRlQ$!O2nAwf9*)ISEy -z2{+hhV;RMkkSc5KPNP9C;-nRWbv_1@Ehf7R2p_tK>UwGd{rX(Q!E#)_QZX^sp*X}UxaUv>)}&d^RI7&;Y1a(+EtLpm`z -zB09MZ?o44!whrP0tbK!#AcS>7Zs9J?XQ4FH*C+I8(Fs={$D$GZ4hpjbsX0hR0$?sp -zf{6y|5D4IepYOgd3g9%@9gN2~f%4sOvG>Da^Flv~C#-cguTr&Wd|%SEyhyq&Fx%p#dMwDq -zvN9ozi`d4eP=9G&I2%dA$Gn$n4j5!vO^W%B4!ZnpU^+V7k_Fsj7IDGr%}aSL2@%W& -z9XA<}fd4LI1p7}NH>r@GsU-{ecXTQ%m;z-^c`AFm1G|#MtT*?h+7rOlFcUiMMz~;; -zpHN(o{pumy?!d7nvX>N(*_$VHbf7cMD=`G_?@r|=3omrsvBG^?7{5-(=?fj#t+=JG&C0*2cu=l0A>qNY`)4IsTidMInjC(FWd^hr;|8H(1K1HDc;tS8SJ&aNZHHR?gap^i~g^sfq@x>0kinQTncx4R9xXNvUD}47dG>g1?I|))ZMSynaz}Q7a9=xht;15l;(vo9t3c?| -zPR(>=7O-7fGUZeTx}vYW+>l-%0$*%-r&c8_`C{nQ2a2sA(#-7+a>J2az;=LqK;$Xh -z)9RIu_X}A3uVwCNTWs}>AOFQ3|45B-uzoEX*6;=f(ttqYZi%~z{2eVjM_i1t^OQ;Q#;Df9v?GxkpA42*j4SZY>hgGy`?3~nd`_u&X)dG -z%29u^JSBEbje4BWh7QBBgy=Lia;q_{c`{j!i3dhS#`s5=1LWo@Qc8yjVFRUj1$400 -z?Pi4dr@lGY}?8Ugye47?+g -z79jG`Y*xT|8JR2Wm6fYmuCqK^_=q0cwggeavzFf@YyJ#ESVX4>lpEP&YICGN|3RN7 -zGqUD*-;$pUWV9kpvLuF#GLzTq6m&nxAD5i}z#QO*RoOAxN^UX}baf)}g_}O!s^v@t -zSTAN$>R;2*{4>y`JeoYf0q&D~Q-#~){8##J;YTeH%pN}hrU|jH*BEvB?tZlTQELte -zqsa6zrzxPCOtL_T(s4nklF~dQ6GaI?T1Uh6+=o-qpHBab4!x59po5IE$4{y;>h;Qg -zMBCKC&DiSrDp#0-wL|!?J{-eYON|%#JDsWXy!_Eu+&F|t&3Gph3^S=vlV<2Mb@-#;>0)-*$6#gV+Z -zIp5SbCWHgF<;JwsKI{S$!Wj-T>dGhe>*EbmdmD}6>*K>~@0@IYxy9(;3c^vpxcZQx -z^@9J&^M=bqw%VH2h{P)*(I?E$#sq)nn0D&i8nuBxifOfK>LXWTXI+V%b0v2EmDne) -z#6Ec?cEOd{r?14u*Qkf{O~?W?GzniLV6IUQWock_jXF{|tOc|AB@i>J9eeT(?Xin% -z)KTo0T5}-oG+irvsckrzd+Jfl#!)p!4PqY7ET%!bsqkh|-!6-qh%tKj_F8URK)*GNtK&Yn0!SFBPSONL154SM&3* -zc2=f+?ke>o+NMw4`sPg51W-}T=VuwXSFLo%8m{BkSn0Bi1wjR#h3SNdmi?6S2nkNI -zmQ9ky=H{o!CHk=wLmnM1w__TpB$Y#i>tQu?hca>d%|^LoqCV*NB>wEfBypp34T5hF -zt}|iW%sfYYt`MT*V)H<}SOgx=16L{x)^Stw&=}B8U=a9FUZdltZ=J7V@c;PMQ7S$d -zIlxpf_@A;B^6DTg$6Fz<4g!h_f)Y#UjUH^aRm^Wam^M9Z5}gwM?exR^Q!XJt986^a -zxKQSscKB!guLnq_A@;2YI8i#l|8dXeI~uKmzm8j)?^v3jIEeJN7ONrgIqK7Dp-Pnu -zAGccg7}axT_}I_+?<+okX8{!wSs(qNHG}3!dE9ppQH`3g%!i!Gs -z(R|0%9NtsMjm~#;=YTupq9`ZN%XQpv%Aw^ -zkVoJQ9V`^IoJ0nJXo1$VhP$_~m@?6(Au{t!DG4~D^#nmbUHp@11rq@42-M9CN8OC$ -z6I%DCT)s__Hi}R42fPfWiQnw4{BqS2}{a(1ZU4fEt -z|2E<8;be|u-?pR_18i1Nc6~U>$gvvmp^_G%vvYw`P&lVyqu{L0Q`di>&Pg}eiq&uq -zF4{r6zv(E=MIXzyv8U*W9#k_{5B@i}0pSyI5DqgV*a;<2FZfbE&?(aIZJi}baLJvW -zcsSvqgt^hF@R10&w8{C~Vhgs}=A@PfG50#lhcX?V<#M*IvwW6-&^W%clO3(qM+qKc -zJxngGXD&OaoiFaRp6kTI97R8-MU&y3`7z8&CYR1ngu40Mc5!30KdOF2*U+C_n^2(3EOCZV=@VqH7zyV!#$eJVA_I_ -z!Hv5N^Hucw`&=t{Rgbnsl!#>oqrkOxWBc! -zl6FN=>uPLzP~`&=WZdU2a~eeXai6+cXw=C)scF@yDDD$+J>(8rDXSycwCA;ZSDFG| -zur~H3w{0b=oZ3{_`VpVj{c0$K4ck+RtoSqAo|)BApN8G0GlE^!*>GvA -z+{LX-&;#N_8?HQ|g8cbvF=;80j%=P-CD&Cp^|&gF;%>SeKWtUpqqNVFvIPg|cUsvZ -zEi6A}Kqse5BuYq|Cgbc@#|C6z3$%`R@V`LGm0J0Hch)GFK%S6~hWHz^=B|rc(g@Wq -z49cev|4`ZobM((Uxil*^Dk8o@_5fW&zQ#^2qd%UtxZ{&0ljEBgf^x~zu&xI^+}2tG -z*l#+Uzvtc>$e*EYf;#V?1Fz{?QG)ZBD8U&KD?AoXydq(Try!o--fXU8(fbp0Z{rAn -zjNeOV2uS8ZU@Sx@lf$6|-MW?Mb`=sdRStyDP -zKYOUNdCD{@uz6j!;=6itnD=M}3BR5|@p$;!yGgDV!_ImVGTJZa+0-V9WKuK)_6KaS<{&C@&wC)^4Qw=6EfkF2y|-c4Y;xOJ$00g0JGvPkxjSd -zIC662IjMQmym3O28?>d-_FiReb#?T5}SlD9(_vyXz8!62}4T`tIRxz^2%wmx?|M-knuKQwxG4^ho}*xm>2iBFMVW~0T2 -z+ZXh=c=jXm#KqAgx8iIKy$-uFl>{UuMT;kQQ{pA;9_XZJ#L7lG2w|#mkV7&-Dt$5` -zG9ftEyk)cp*b{r`aeKT30i3zxbBNX&`c1Mj#~Lr4SPt>@I?YxR4$Y8Xbnw=ggn;&h -zz_ZEs)n|=I>^S>Xz1DwnEEv+6$8w*HUbBuUB%0|d#$d?~t{`2ecvn)pj%ceZTS~AJ -zv&Y+7i~)_tKt8xlc(l!)Ku=Q=0$#pK!HjRO;Dl}AxOPRsH-;8+$q(|*ILLcwt?x<)+VFMic_exJeTorW24&I655Uag{RUR -za*AxXi?F8FXS%w(+Y{8yFJiyw$An0#6heMzgOGO=mUg!0dhD_Hq*h!OcJ~tB+)H`s -zm)ngfRh{mA`C?v@t3LUB-u*k7uG_a}-o7wX)qVS7YL)NVc)>f|o)BbyLe}(%#Qcmb -zZnQ-6gO4eAzKQX)f8D#}92Cp^YdVJ{ui@LgecHey#o?#E&AZ&3cgDAE(Dm4}(O@0! -ze3O@aJg>_y-8&=pe)Q4U8e>qEqW)mY(P-w9YfgMzI2EeCqC54Q^>SU_X>G(ct?EvF -zUh*xgQ!Cq49L{}_m-|Itxw6owNL^t>eU3wU4F~gTRY%D&<9_u~=UK_7LXCe}mYDTzAX7%Ri#5p}L5R*G?q>TZvf%zIlkR%ZqVEZKF*E*e4Sp4e<( -zc05HbrL6F>Z`^b*IWMnDHl}c9K{==losz2EILfYSNXn}}SPdU+iJ@*DBKn;?7))p8 -z!JuhVp213ovq(A&C`s0>&m)R5;Ge4gIT&ZudhE@rQZaYYe4=~$U`vUrT*~fhgIv!` -zxOI*3X`>%eqCf1fVX$PB@dx4Vfy%(Y7U@a!i+NNs&fX2H@X$rh8*82c4-udSQzSKq%R5+ieiGh9YMB9nw9uM+ojAviOyGME;~F!C43P862FF%$ROFm|R^w -zu6pbkbIo-k!zC_9J?G7me8*uwam;h~;qS1F6x;#U=g^*tyk%^B-f;Pt81U4y7{DJaf&3EZ`i#Nk;fREKpqcVxa4)ctVC30`Gr#DR#Rw1p*lbNXlR%l^T -z0Ogn`={5z-FhvH4*QCE_TA5&)@i&5l$zVC4A@fdbcMQwRW%Bl`!S~?iL*j8hfJ%S) -zwy@Dk^sOBVJmEK(q&7^qo+5ci0#}|6oPXIb@6I&qwZNIq!0D)|-8Fqt3(=(g?8*(! -zOOr@)gYsa(tsyetv^4G)*V7-QY|0zLq=Z#SO~J!WOswgNNv01+&{j14$CazZxQRJi -zam7<&yW)~7SCJP7&(ndqo_R&;G^r$4j(=1sQ=D<-%JLKsT+eHb(mZTF|I^WYyh<*- -z(PN4t(Doa9ts{`J>7L8jV-@|yVC~P%hL559xqRsF`wQ$t?&Z#hQz0QhT$brNPG7m- -za+$&X*=QV9b8t6ugVAo=sz^o02PcV1}yC8pl -zE&qiAI!7JtP0_pffzKAd8;RW8?E(W -zHiT1{%`qRl#!M26vzuvgc8e7@)$SemKnyMZW)Cep5Q7~+9fT*00w!quQb3Ab8t!(W -z_WTBBuYx)3q!txg^h=}%+&v51*~ztAgvCz#M@cn&l3>uf1FTUi%x<^L<-c_TjhXM9 -zol#21YgTZ%<)R2s%XV}!VunUB!x1q9`^OB6024NY?y$0{MI6SpbEQx83)V6HetjtK -zlOSOichLhnLN`Wt7E5yYEc-x46!=STgXik#-4}iB3}w!v9OUoE@3pp -z`Go#m3gh-k)df45EIfqQx`YC{n4SmBDATBf+a5d$yWq~I -z#GcaFt?{0vvTFAK)eev2#b$F1wv+6HF3O!~h4IwzW*i=TOjk|gea%lqV<(L$Gao(| -zHNEq%pv=XW+Z%hA{8dBN3KMC~6KHm^%uraUCJCbv$w$(*5A7P2DZr+6E)(8`t)Yn~ -zgwlr=4<-Z;7UoY%CYZ(r(og`6^QW)A^nw0GQKQ%mwCo0E7)>35hcLK{EEoQZzIxFI -zwU1?eXhIYc_&Htr<@{4!c7D1rzYCjuoWfEHQ@Rv5ki3>3g%@Td+RToik5IJv1KM0I -zi%P}zWzzO(1+*oK(IJk4S;_k0L1C?AR5V1tCVp2?&^S-}Dx(h%+ErArftKAM&5EFoHQ+WTbxH|00g~h0n -zT}1PvH&L{CQ6ivuu?Nr5;RA@248W^o|AKbYL4#rz4*?URTfl^v!l25v44^%i8A=~$ -zryH*r;5WqAz{dj&+OvULp+r&CcF#H*7saAnlzBXf?hbNE3X;_79@%N4uf~Px=t29G!tQPRpWPlz=WqQ3|02 -zl}G*J%#zR$N@#iXPASknG54KuFG146Tyw_3B5W=H3q`#fbUA9Hk25enP2%bL!v07n -z=NTtFNS~z()hUPq8F8^WK~ulQL-+;{<~R)8 -z1CtYjwv<^l}~6h8q~6sJfkjxH~!Q=S4w0EjjSc%3WYb*>;q96pPL4yT>- -z5IUWtk`lZP_$-F=H=1&XJ`ks+1TTsQt&0-;D12#QfD|=|QyC?+BE{tI(h`@QaWcSM -z__s4|HDFFxYo*YFJPd%UWf}nUcUTHW=1W>d17J?!HibWcWC0;1^Dp|?hX?-|9zqQb -zIcQup4Sh`G_R`l%`q+a8Gbpx!mfgUAMN`F^QFb{?v0vCl<36Gf9uGQl%NfBrDvJ5V -zY3H9tFQ7*MiT2{OQ?UaurzHJ=h+2X_#ZZczJe^ET1OwplKq1L*wQOD5qp^ -z(!uDWArz({MSlTGr2ur*GGYQ)pDFxtXB^)LzREOXq$yyyDaOYHUzuSfpb5J}8Eo)( -zWr@f8o(a7m<6hiM^=$lNY|u!60!|Q64$(j3fgBUV_!@Av!2njRlbhC`nVAsAIGmO!QG)|(W%r(x8$Xad -zE=)Kp)&4T(d^^=yJnGcU>ZYc$C#3xOu!+S!ZJu4yE8#L{ -z__jp5;%8Ul6;X3^uu*nu*V%|gsTnci$B=6@>V%%)`wC*?vE5G$Lq*oISbCMwj>BIj;r -z(=W0k06Em(x!x!G*)9G1he&!T(mk#;e$hZY9depC#V7i&593z4_dD~qhq -z)@Y^G=E^CiTm63io13-8aUcWdD~E+9d;eeQ32?|pg1y!Zk9}+li7A)oq{uayo`g|l -zYMu=s9QaE_O=o-9cB^M;tAm~Chney~i9#l9{9EOxjV`WU1ZuA#=o&$e^j<9&C2J -zn*l?|;85(R=5pPeMHmY~p>@ji55!8Rc}_-)6koWwyZwm)G;y(*xD2PWnxKR5Ogkl> -z?^QC8a{OmA@g6Z^^5l%7fiaH0xOs#}b0qQ32;_%pD`pPF6I@;)Vc{U=$IZ{qoa6EI -z^7<9xt%uCeFrp~YrswI>GxGj5NyVS>Ta!luASzmuXPK3IX!Fv^(vlQzQbzyYjqwPr -z^|0G4!hR@Ig|3IVl~BKVS=y8Tm~0-g1UQr|Nl;oaTr*4@Bltmq*(4pUJe9hckli-| -z9EUce=x0}%e_x%Zghm2#rIb(JFB|UX*oOq>)2sXmn(#>y##n3pBx&~*$omSY&^hsD -z^*R5OC*d2e|Lk|8nHy7Utci#DA--le+w`h2<@fmAATsf@GQ0 -zj7`imaSyFYl`-bOW%}q?f?bXC6u)O`kYqf4PW^|bS3@QAxq(}l0jwo^`Sy@Q2e@U~ -z4t*x>u)7n2Go;KFKbZ9vg(+Z}uT?Tv@g@|8LC(O9%q$z?N0tMFgqeE&d@oi>GbkO_ -zrPw!Uw^MArh81#a&{=D|WT8}o%d_2gt}TRfEm1xjv-m791Noz0rukz6&&I?{BBc`d -zje0cxg+Ip|a9o?Ng~q>brrw6J4{wk}HcTh-==qylKZDVHb<(&@>H$f2P?A1`-fu9h -z_!kgSsh;3`Iz8}ohPmg5;k}6m|Fa)Bh$Y=vrGJKt7*STa2x<32F?)MBSGEZ^uti2U -zJd$mSnPr->$h7cj(@Hr6eR9cuY3`WK^}Fh+91_a)B6ainS~ZA<48%R`Hd3QDmj#Y% -zxSM?{GW$s2$kTztAu{Z6A%WRkf480tN^HOVNNU_2mn~1MW4{!1EjMCw(@huV7j7wo -zzi9UA?G&49la|1pLYf6H+4Ey*r*(giilK*+{=JS=5jOuOvA -znG7v^d@g}~0P$#EI4X_=Q~ttJAICoEs*>}4T@Z^RBx5?dAd0`=RprSz@erKYwB3q*usL)Zq~~NRr!^UABMHWjw!-r7;pRv>lhUklR(MX;)pz_p{4tpUuT{ -zzbtK{1S^;l>SRq*kP9`CN63`hAEzq9B*{0k -z%O=Sbk&@hB#A>F}YSxm#wBKBgYuRN7RSkGfm8A`n#7QNmgzVE=pA$HvXt$LL|MYZ! -zG3~JHax2;weCJYj*_X5>Dc^K2NykxGkdVoe+@G?`Cd=X`NgQXh%N}b$89*m(pR3_? -zcG;mfPW_l&Hf^u@)j0(G2Q=Dh$>r~|%O01-f$*QoF8fPvfBrqLR**d*)Z)6=4%DIn|JT@?8Cr -z(!1tY*SSBLcF%RWF1z(c{gS8J0XNwIfSWeEl54Y*GV7Ndy={JV0SW5Qp5e;YlxOI*O`0sE;kp_g8h!ZkWStny3-a~2p -zH(XOezqe7k@FU6W@pFla_@I4>_sL@qGKjWjbs(GKV_)r$DVOO(G;2kq9ePt(uyNBM}pweLb+mZYQ?6;BO$^){h8HMAGuZ< -zwB~Jgiqd+V9@+APcCURkmDjI%n<(w8)sd?MYKEzwz8w3yXUnj_v(DAo+{iSGO+tr#MrrZ8u68o^=I)y+CM+@4A1?>(F -zxj@g@Khm1jvLZca-o;U!8*WG3*Z3 -zpvb8DUs_h&LXj&wBv%$maJG2lP5%H%TuHzE(FvL1QZ!L<*Rz;}M}_hM7zQqzp(rWjaI^dx&`6B!8K -z-j;G1>%3bYz_i~jk7RD$EstSu-Yu62_nWiQQV@>w3{yDJqy&0c&1%P!|7VlZb_GwcsJa!l#;p-l;atjHrFczuduu3wXGo<=&l7~`rBj%

!R3X(G8xwIl9?N4~d5myF-%z6gfFJp4L -zRc|U&Az)u^{xJ`NJnVu>HB%ccywolWu8XhIVoHHl=x`Fzv${MK(( -z8PEP!d!67c5x1?RtTh=~ui4gV!CD=DM9J8a)$DWYJ37;0tK0?wP92-(&dPRWy?#W& -z;MZ`>la<|Se!b0{y|mk%4~M&)uQ@-o&mLA<;K*84IBu|dnRyis+{(HD;Wv%Tz8F9+ -zSYCIVvpwcj2Aik$HBZ*6Pqv+oH^1gJzut;6f-jaWhwZ1ug$#1 -z_Cy5AfsiBr@E$p^v$O3ai6ObU2fjc~-!6q1`L`uUc8BorUS&X;R-D~vz*%*V-JET( -zZP>P}Qf)E6ZY%_5f3Mwvij~`4&m?87wmV~=efk9~ -zZ)yJdBTa0ob41n6p`oM~n{1$K+O~<>{5@6Km-DxV?>cPTU=X)?bw6yj3<;H1I5Lk?Ah4qScb^im7V(&P?)%6*v#u -z50$!hnVh@L{HIOW5C1Cq+D~FLVof`%|73P>%w~Io$`NRHO!!mM1iPWamUbCJ4`Y4& -zq-vZa%T$9^!l;XLbuBPcuTFT+1a|Dui?k8JKpS6qVyRB#PoRtWw<~p!e^WywjCOc> -z)nnTvCS9jhGUaOQ@D(RmHTS}alHIdTa(})}2YUUc^0hm51D{uUocSrvqx%yoLi2N- -zq*GYEv4GxtI@Ule#N)!4&H?*#s|P4^ELE_N5oXBUo!U!(`ZO~ye_sC*WzuIIY!7V! -zgYS9`Q(UkoLRS>Yn)c26q)3sLB5`%oQpy6rW;<|VsWKAs4mxrFiMe3^^ZU_qHb&fc -z0*Cg|!oCyD0GUe7HQn9B;;AJHm$Pz5*3Qt}S2mgj=E8uJbP2-%h|zNQX*dZV@Fn-V -zWWR-CxP7wUuxVq|LL@8$mLb*b_$dpvu*#xv-9mRj{NXACA!`}#ZiO34^GjgY`wD#<2?H_DQ?ah90FpMocs)U*mTiFTZd5{}W#KLryK^T95bOs+mE?o2h*aXgBwxir_^`AUkl9l*hM~tv0{ri{ -z``j#1V3%2h?jAXX?k+-i*L$%}b^|x!{+oVzsp9x6kUadfT)W4hvA(PP;@wC9@bpGk -z*2Z$2D}*_4ZDJ#}Dkcrq;~MN)o91V2g#44TTm=U2h|4xE5O3L%O`s8=c0*ZvepW6& -z0mT~ovBNPDNTy|jKQU%qQ{3&Ac70!GRpgC;wC0t6gT#QX_h8iZ-95WG*VfBajvj6| -z1X$mN)Z?CaLvf}tZ{&j8sV0wklMvZnu1bYI= -zC(9Dl>$?q!seO7~nwr`-{BBn26WwwZslE6ep{c#+6`NC|<`8`n?etWe4pq9dHka$g -zOqg0jCDkUcGl%WQ^NnuvW`yX)agBAla_sG*y4@EfO!5T@!DtVJe@=9wegmEZh-}xM -zgB#4lXMOkWwWc!Vvg(k*s!tDCVMl!0BI^q_>+%=y{wrCc9zgWdX!iam;eD+K3~b>S -z;gQxY;cVrZ@NRgHDLlyd#GP=parAmU6YtDf&oJ@tLyacmi;y~vIH?gr%6O3TfHN2O -zftGY^x;=~45)-c?^Mt6=?WK@SFk>HElJdE%2 -zU5T2w^fR_xw`2|HQyYLPm$pMLm -zfv#hxTAP$L`n1y46CIvsnY0RGGe0Y_oS&5rLeHwsy!*`&J%jYf_5Ys5Vjk{2t!!0P -zdz?4jP+OmXyMXIhqCK&ysv@N7dPrMvfIV27(;13~N$tTJze6oMZr&V*S5;dRt6N%z -zwFT?#!8(6}AipE9D`iz@sGQgn+`Top8?@k|`Q92F?02xTqpE5cl>gj`o_+ggi2^ZY5()Q)Q%db!}NsegA}V{20a;an$TO8fw@WkJeXD@nk&{7w@!Z -zCAhLiy0b=9eH7}he$JgW21?0UqpI9a@A12>$2(fHe&fv=TUAlz&Km77kF=W;TFuY2 -znMcp<)_%PxEXZqqW+Hlf@vnEc?*OMCBwq7xgwHzN<`FJ)!Y8{Td_nx-PLNp5V?5@u -zqNlScENWSG!hCZ=v4*pB)CA!AzYO_?HfP4-;GCx{d(~OEPG6h?fXKh_`!#l4^$82Ml6+{7~#rF}B_uJIL+D -zDanP)+VPJKySdXpxtt%m)j1yZTH*KS%M@b&wu}W&xcl|}LGM9^F@}rj8`@DM6Qi0Xkqp0ZuX8 -z?;jR|?Z+?q!LHEetWUJ|GSl`W%$AhIAT9emWO68kpXCpkLsB|&dC$w`%@Kb`<>gU% -ziylTs1p-;yesC!QoEuOeHeqDX^k<)e?9TW5@%pB8(1y!Y{N^*x9f|H!YWMLfN16ht -zB7X<&qK-5rzK%4NI3IR*yrzrNp)vntr?xOxXZVT?X!k=-QpgOr{eRc9v -zB*7x=WAb^B=7^grw|z4=4XMN&w4I;(uyB{lSts;rKnG_3!1f|~Uy$qAP3Mkvv?~VH -z;&g(+ncyo|oa8TeEXdL5~4?I_4BHZ{g=f`t%6ifKS{>N^q} -z&o_P(L`!zyMUpjtL0;j2X$?4PlskW*_qz6Y@mJ0M%sX -zVP_{!mEG|79AjfTnSRVo{F#`un$FIxj8fIv`I<6gH>1GcXBfVmmp@Z?Q+N9f+zO^x -z5zU0)FHZXH$Iz!xVWMQAa%$AZv~)Kr#^Br8`5XMIm^T^b0vKA%07VEBh(C_059sWC -zli9+A;?Jl=lkt}z#nG4kDE(`SS3jG=Okq}j)~V=J6r(bRQQsV(LcNHc&#=zU&UsQx -z7BJn%bnoPu1cy^#ssFe5-sTLEM!cVC>Mq1tff#TLRI>FMc4(jw0ro4 -z&zVj|=mi8TRe}FyXuy9Nswh@2XS7U!Y8jrn>wIPz!wh1^;CGnxJBAsfQVvisR4iJ@ -zjA1%eD)LhZ{C(l?SiVHIn8BAnLsZCJMI64JokOJGcT|YGv%?E5d3=-lkr##%(I#v#XR>|OalHrikbdc47lMrk78y%iopoIRik3MGZL(0 -z?%xD#(G0)I1fj#yJj+l?BLQHOMh&nmjSPT_{5klO{BRT!R7vq665NF-l4;}ulklr# -zLI@(@3&F1vbw=TVlkSYltY99s?y)g7BTd&>KQ^Lf^<&|Wji)*AvGB)6(;V_x_+w*f -zHa!;p*hrca9t(eL9L+h8h5vdK73j%pswY$!1Vvxzw~zE|lzt7;Z;13;%)HIq`oHYG -z2YggT_dh&$chf^UB@GgY5UGLC>k>L5gx-skfPjbyNGJ3nAXSPWU8;f<0TBp@f{KVp -z69H)=O^WoEeZSv3cXn@<1oe4-zQ6zbdH=JY*>k6zbLPyMGw05oyQ^`=N*RyVqV|?w -zhu;y>zjN>X(Gxa9ixD2MAY=A-fHnEQ;r~kD|4QKhT?ue6n?XagAleW;5Mi?Ppz@hk -zWh_wbn4c$;=#RVrMB-HCp1Afk(!ABkhRF7;L#QJ2RUrM$qACw6AjD-fRB(ke1(tY<}|Ms`^0wxptV_yr@5{r|K$s -z5xNJ~F$dK~@nkMj*QyO6`>5n=^Yo@lJArnrqZmZ^O{CK+Z8w$ -zIFAAm>B(|l5ucJX+cTS!$(ox$4s;z=-KOD|`6)?CE*W|eh -zBImyf>>TS84{52f#-$$ExBs-9Z5b4Ql$AM{?TR^HT~p5gO<;c+&A~lg%P3_3*-z>L -zuT>gm%h`XH=R>5NR5>m)TniDyEN3W;MvN`k03pN_nWo;DBlFB5_10s(<2d}V60aIl1(|P-ioO}5iZiJGvX2Z(9tu*(=BQUPeX%{u -z**EG6`=D^)QV+j`V=T%F5!DLzp`#Eny^$Uu#GRoiP8H(97#Uf)7m%z6f@MJT9wx|q -zDx4`y$G|K@{2UR-tQ|)YO~v}9lDm=v`>q1-92qQDtJUV=B@?V@gt4|7W6iM!0`D9Z -zo7<((V=hjN3 -z0kod=1z@VE{nTQo9wY+-1kKc8jF^P_NXM+#>j`_DOs5_oA87Cp^JiNSoeEj^fWkZf;MG8qDKRu;s -zM$*SfLX8n}A#iU80`^9-AWy_W=Dk@i4 -zc=qhsBlWZ;z#7POM2<-G*ga4S%yARa<`@7aHb>RI2rd{^+i|#NRBg}Uno+f_06dIF -zNX>z{f~eyFp44hGs`jbhB8{ly0G`wbGOG4=tU!zkPg<-HjHj7EFf -zAqIs`rkIxf$vQ3j<0^Mg%l^B{-P4LY@tfLEqwYETqw7;|S6bIklYDf&j5GCz{FwVk`X+g$F3|PXiJvU# -zeUkNrL0_;Q_BN$H_Xec?$3ZsiBjqFMIP(oYo%zms241w#=@|5#bo6}F_@%YcIB%T% -z>GP_rE3MNp$w!}4*G=W*^MNa^kF806(pKv8&WYckFJ*v!8T3V3eOzM|G3X0>7hF#_ -z2jiNfUGD>gTB-UjJH8wq*-uXO!M{g@y2U2fgUzbFN+ns -zuL1tqe7t@AtQMg$Pl)g*%kMY+bNclSm*$;IK0W+or{mXuy#8bI!QtN?_}a3!*N*C+ -zciJ*{)Asdq*B)JYZuyR-ixez|n|%5z`ROL2r)?NJ)=y -z8J&>VbgOTRJK`~R<2*E9El>;7BDK6)K`l|st>x2R)=FykQT`CkJT*U~JfBtw<Y$;)>XlY{UYnh6d?G9P~fNNm44zTXGX0tW4t+qX~ -zC5wilk!TDbv8iY#nu`|rw8W2(NH3 -zK11*siq9~7hC62k1pNu;eLX1OH`K(hBv8(~vo{{sII2fIC8P^PoSJEaP1r;BxqjXa7XKurC -zX)So=3asN^PvK#{`P$XWm9O&L@u;*rFRH#fRCEYcaL-d|maA(sRPDGk=}kx`YhG -zNBW8$p*gMeRF$*LT%H*{g#%S}dge5FF{jD5IZb}d{IZO76u+6#Q+-4=y`JhL)9&@z -zA9H>8a?)lU^1<+*@T2IH4zJDhh)?y0`R??YXRhyF&i3YV%16<|Wv2Md@>F#R8Qk&z -z9r>8~Cw((NY_DiA-yOdyBUJ5_yqSJc|5&c-Dn8xuE1ZN3W_nC#Honwn70kNGJkl|j -z|6To8_(-2PRUmzH`k(4Q`B5?;&dmC+`b-({nle@LQo2CM;I5D4P2p$zzk^@NJTrWm -zwKvyyFDDMgH?Q48n?mVT7?TlMkNlhC5acs9F_jYs`^Eb6kyn=R4TDYqU!>0{a~%%0 -zQ{^hp3{k8H;#KvSp>#qoG}6pdfpyJkMN^fx0e^iFXCkWWWw`E%ILR%4uc^)^)AdEu -zHS;?o4naKPc1@h45m}#g6n--;#TTI^$f9xom*<@1MmBD=)F7ce8zP?_@Oc2YxBe>a -zu%=s|S)W>;SRY#-Ssz*-Snpf!S?^l!SpT%%w%)SdwBE4(VZCm>X1!{?V!dqr-FnIT -zoAsjgg7v)hob{~rSL+$;Y3nKLFV>URpRFgX$F0Y#N3BP!KUoi34_Oad4_Lp2H~f`# -zk9C)IyLF3ogLN(bPJ4lMhIOp9pS6Rvx;4`Ji=~@IA8hce3EnwcadT<03hdJJn>oC< -zWG+K7QA%_XUG?jYVzbDug~LmqCJW&?))m*q2ii?7-R*Nz`#`&nQu>zW;BZ`a40r5L -zFO%+-ZprYmr58wFmfp_sizD5srLv_5c=%cj#T~&jH!AnEOA5FuxEFFPaNJ0* -zm+qH=h9%va9+5sOy}aW)2mdM694t6hxJ0zke%7A4QF)|&qm|W0h+O*YHMe=-_{l!vXcmbbndxPc?KyC)xL&AmDN`4r9<-V9Lq2YHa99>7_I&3!!dF$`=MI> -zTd*eXWIu7^-Cf82PfBMk*MCbw{}XsWcX&D)xs7(lS^0ljD(m^1wLECD`5edxR_lF0 -zM&2XScJujgXT%6Bx}0}^ryl*Rzf+G!(tJN{M+V~}HMF-1O2psmC%m9q_JC~47$ -zQo)RmdFC|x$u_J{T;?()b-#l6=yxhQtV@4GEoMYJ=7hkH{t5iC15Sj? -zY_=jc#l?%uzo~O;$-|;z^R)Y0B8LXQ6g#zt)qdh)dX)s6f|mGBTh@8>_b%agG25h| -zLZNBd>*e|(qHgpTdGhBSA9p@~sh3j=?Jc4ulqu1*U)qgL!dn0?3kY>L&U)i!}>%wg>w_DVqNk?1fbzPcwecEk)_i{ZCztQc@ -z$Gs=?$=-KCzr61(8<20%!oj(QP9EkMF1c`i+c(&|R*gunQbr<*ebkgpRe>`~Z=I!Dk -zL6)~mG|IcV(afX=eU5c3HzU`m!5dpG-SN}>FCBNzT<92Atz?W%3%k^LU*Bcb$294_ -z{FTk|V|-@a`0a;#YrmR&v~lS^`Ga#LcpmS6KV??4O*LC@8MGpNcG+>iK1lz4!SOHl -z?XCS=i?Q8S46u8@mdg^Kuk3+u{`h&ym35D5Pk4Q<@2W!E?zK8ncgvvnyT28Y?3-9V -zq~Nn(PJOrQ$%oele$e9G3fm*reAIK?kc;)cYLmCHwPFYVs!^Z(@!qrR-+zC);KzPD -zx=pULVD_-jdL3 -zutx86Y}V)M=I@rToHb!guX_!CD||HDSLaK98Mi2Gge|3St -zr(gYk^^SKxzP#X@$@zzP_3YfHM!h8!M-@IBwc9_>&B~6pKMeWxgLR+p`uV+w3v>1= -zJ*;W{zU^id-B9_*fLpntmoXK5_rLVUx$6ZMPZ=Ly{K)$FLQg-BpFM6z{Dgu#<7+kE -z8Q;I{&Uj0Qo$<>N4;{HX2VujTnla_u4SJp;!7SDbIaw{ -z4pe+YORV;#mQweLCX%O$Y0-y8q56--hAuU<)o*;Mt?Ktg`_8#>-CU%E92Ex&;|!7= -zM+#cp)dU`dxI%L=H~a%}px#lDk`k^h?cGZoHDHG}eb6(lIUmW3G{>ioC64Kik&d?< -z?H%nqrzY_A1Kwr8|0C!; -zO%Ddma-iKFJd6Y{OTbf_VeK^0sDo8Z?BF5hcJLL|I)sZxuaywxI=&(tuXPq5cbp`4 -zcHSgPb=@zK%KXL1DH9xfu -zsWm}CwRUiAVNmI;AXf;shS9qCL`HCNfkLxaRaBVh2)?-k3nPyb<2^B6}I`@#6M?CBw -zAtM9n^IAdXG0pl63NQI$AYGO#`U<+^V_HRAI{RtvucES3xsh -z%u_h&uQ8C`KZ&HL>hoHq6;$K*S5V<_m!Er@^%xXX{do~&T}7XKF%X(%r0}V7h0i@- -zmAgaYV#q9?xZKNGSJ6=AnWfpzoUhucKB;t8^|C5g_1&T3pY6MS$&~Ax -z?TC}}lR@EUefBFW`TkX(6<+sx3ZKeXX?LjjP|&@;nimSHa@9|TPxVWctNIG6d<9i~ -z)n4VhL&cY($2~BHte|^)g+D8(@T+!e{ZVOmsOl@I`lt9)P?d*DcAa`}B%MHwAMvm) -z>AT;fseCqaFLT#N(q&{mBm1o)^NTLS*~E2NgFh0Y!M8#*{!@r9qlCD3Lx`99;+!GY -zWb(p$);31uHS<`e#tlh5xYJkp&q18T!wqqexA67$6JB_U!Qb0U_;~x{z@Crr%Yc@A -z+AYKMF}8;cbIuhrtyI5Z`n*ZYCZPy2pg}k_RJKbCGur2H;Sm*q8&L%tuDuccjl3|x -zT!>MKshFOrv{lfBG{@$K8W20H04`w9-EgK@|7Zp(LSK!haK6{)Zo;(rStQ;x+ -zC5oqnoN6^U##xAqZI(4&rL%n;lor-Ed|>XS`MSh+;DIz}nD#hPyNYSO)icrWy?_IO -zS8^T*Ig~FY=cJ;Ek@d@om=aAz{K@u-&Nc--Jp(=o?G@fH&jHM;Ed$NLR%V(BE#CG=s)(6-=}_1%bW|N -z^5^x7?^DFLluyNkYG(TTp5Psurw1@Di9TO>rue@Cy7$6F&UbQ%@Xir)!|z3=M0n** -ziH^xTH~N>nBKJ@6BKAm;1Nm+gNsOzPn3BI_DG}ebOiFyYa&uokS^hx5m`aI-9#l#! -zJh_S}l2|LHct4M6LDRy!#-5B{mJn6$aMhpd1U53$k4R6{LM_584)R>*b0F|>wgVyG -zgr?-2m@6^zl^7B8FhRtJl}#*Cut8#lb6rwi8S}P&oZE@wp*2@<*{I%erHV{1m7~hV -zI%gV%(ab#;@*sz*LR%D!s0` -zY3MjUn#1C-Y5Fq6Ys^}FYapXTmfIY#E`acFDG -zj^JZMOAGS6G^oCw_Q1C%@u3W*8q7Uj^tZ8G2Ji&zz>g2`lLtb!@&abGhmr9DelO-C -zpXHiSPeYooX0YFcY%d{~CIn6B_3>y3`NF7&lwOYzPm4$ebZoc+B?A%<(+Z+@NaOgK -zfzQD1n)JKlh1iAJ);Lj!8RdjnV8>a++BoyyP>5=+an3&lbF2@}+@}h$bgB@qF2O$( -zeTKc%1JL{5D8Q=8BKB+}!7Hbji(|UO8TdgJx-G9UT -z*YaAn#62@^?wsZ`Bj#z^Cy!EF<+`@y-LQlCCYRoHA>m??`n~tZ_#E5$c3Q}Nug)ny -zPkJZYnvWijc;%PUopZDqTYG7vz^%jbR-F)4GiCQ%K99y;c#wDQ*Hy0nSf;?GdaW+L -zF=6Jr6YtlyH2*;S@#VFaW9HRfIB4iq|6Q$qn7zFE-NF0n#Gds&*7wn>Ut)56HT~W# -z$G{8kD_j%7h-pijLj?(%8wlFTDQnm3L33S7`d-ng+oiRkMEnXR-Vt)!L7-R}Gml -zE$_YxzYV>3biLo^&Z3rM+|`+H{Q2kLU$>69kmT7mx!Q;ar6)%}+%&a$)F17ej63t_ -z*05Ijv$cL?9T2l;Hb^^F5sc({9Zh^xNf)@7KOK%0Jtz*&n<-e%PcfLuWKx@^Vt2@agleEZOk>#C=QG -z)*kS3L{y&Y)eklvw!5y+rpCEW^`G1)%F!>WOsBbPyv7b(6YN(jzW1NKMuxst@Yb`_m*Vbb@13t)->N~QkF~Av(}??d -z9V;51@-Nop;w{IxmhHT?*9J}hy#MjVgYB;lo_*)r0YAL4Deyv`bsvS6E6~1~*6e|A -z*Y{Gwm(Q+g3ApL7=b9YR`$YVO57*B*_eS6}y+K#oXT$ -z4wiiP*0{Mt&lF$%=BuyW@9|FBg)iRhG-Al0T`!gTBmeYaC$=rURcYZ*XJVGOosgE# -zF>lwF!QEyAHGFr~N02W4wn;Oz$9wjaK!b>WSEt*hp!_ea&Yj(6U! -zJ=s40>C=sueXN(8ABZmVWKiProZ8ud*dphA4)ibie!_-BEgmILpZk5$`4`JKc-^ro -zG<&ys9nUq-TeVEbb0-J2-`Vz;CM)YKd)B<;&}RY7(t7@UXvgm5l^Rqnzvt<=Ui%j3 -zdonC($*XT~KYsA@_=OLDjyu_*QC()q}1urujT$dUzt7YHw^!6;gsOjzfZgT+l1T6;~!3L -z+@)^cEw=rQwT2&DT{`!A4$n<-Ti4!yynjr&HMw*8_UYspF?i>;-QQL{-k?C!i*3Iu -zziM-0=^~Hn?CKc*OX}&ElJzdMzL0G}r^M5Z9#!ai?v=csw(#^={o~l|g>EfQsk`e^ -zRPj$cuRMKtNa~xDOYV5Syrt))mEVN@kgxdm#D`zz -z89H&>(o6Ths2TCN>%j^)2RAG5`iFPc6ddU5v#9l(jn`N@?C^cjra> -zROnLZ$x-iOowgTWQzfk8yAku}j%c`YA+&|EI+k=&ZE8g7ss8P{zm#o<6lqi{Oz&)K3ZV66+6o3?NWYBxQAcAahtE_ -zIn`iN#98~Y3Z6TfFKVMrZdYZ-gjR39-M{9RX5TEVx1z<*-ZkbtN$Ihw!>Ku+?HluT -z|87g?ymDkp_u>`3E9Fhy@3-^4kHd#8e#8Ijk(ur1`t++a^VP<4*A!bGG`F;GU+dh* -zc~X0LH~n>}&r9BYuN@xwbZBzOPkCRCygMPbWyQE`hmRFFf4c9V$Kz)%{Pp(Lz72lC|XIsV#-YWCMkztiDzwmFUmzXf$ZQ)O7IiayQO -z&i*i4;%CvbKYn#)M7JqL?|g9KoTX`nz!`mZm7MkAstaYOOs`%uw!`EpZ7+l#T@zYp -z|MzQ_PTde%VCrinzN_`)2X953-csz*tz1`r?b)_wvzPa;tDk#l)8mJReBUf7`iszI -z&EAV^!$yGkLD*91eNa4ZNG=LZjY)P{8m`6JG!?ix%1P;xysJGJAOmepDNy6 -zxHw_ngV8<06VCh;Qf}YIMXi>sZME6A;j~I$G?+5D+Kv)wm1fQkuUKtb>-52mK6MN_ -z{NA*?y!)ZY -zXM^6}z3SM=+xNO@ML+MiIW_eQd*^AB{H|QdogTaK*sJ#0*{`(Cv*#V3;jQXT|E*2* -z_^skGPmG$S@o2m5!nN)tU!Pj8TC -z%if)f3tZj5rv1-7fBiVAxxHJ9iYb00Z+Yiw(RXUuFV9w5yzOG6pT+itJ5T5SX~4&| -zrp&*4DQ%8tx4Bg}-!8X(OGKUyPwKopX~d(odq3-vzh;?E&um}3c6srGGM%R$nbo0p -z!sc})zIb>wvQEKT`-}Jgx&G&8e_MU+Lc=kGTJ_qobKmb(*LFOA^iWFu_HUn_wZ8S9 -z>4jduG=BRJmB#KVQ7JU^M)$OAZ+_BZ^U~;~bqQhL-x?QI^-SllV_xa@Zi_|9!|!z% -z*tkfUlKl?P%XOjbp^!w&$P*7X?Y_LuH||co`^9Q4dirz8t8uZNZvI%L$+hM8z2+?{ -z9o^{HS8f!2IMpjPzU@aLJu970om_E9`wrhuJ+ffg?y<=??6x~?jz-<|2zbA4+TNm1 -zU(Z`^;HO0gtrd->Xj@B6C->c{N!^1pkt;&0&#esUYR=i!PinU~9(FWRt`xmFo|2Fc_{Hv2D -zZm2VN=dOF&%=pG0zdY;o!TF7C-$`AcZ}dkkTEE(R;+Ko+znRiDZl9&&%GD!pbuZnm -z)q&D&lgBmMc=f}D8{2(0G`PaKf!(7f9KG4+ldAI^H8)N?+_YZ7MHBa(syVgr{0ch; -zCsa*uUB61VE{Dez$o}@`PCoJZMh+X^wfI*lzbrqLYsQ!jCw}U&VbsnlS1x_f`{0iG -z)fx>iUZJV?{(&Cry>{Q5Te4KcfTBItrHn20&}&7h-Fp?w3X8oGIeC+n2Ht)u>cOSL++>9M`cXO4`_P@NvyVTwrU-lS&WLo!y!&i5C -z_rZcOzwNB}%cZT>s-bl{MTvc->wolKEBlj;wPPOOhtzuIYd1qf!)h&k#qq}* -zZ-@SHsCmevU7y;PERQ*LA}Mj_=1&^_G_s;!QrRyju3vcL!q5kwo``Sg^ZoHU2ewMq^_jk%)p7!eM-xhQ~-BC-MdAv&7!}proZ-?JrzO+{O -zoGHES)(6#gO&I#;tSW!xs(pON`EDt#Y8;)g>i6IGd&O+;vuc@F=)P!CuGz{4UB>?W -zq)EcbV-3#O=Di#-?#7eCej8S_4!Pf}aktvm#~*vtsBq(Q@)v!5-+6oRkJpnuw$^)g -zaTol$Ti?C+(U`bXO=4>X#Z)_#GUM!9RaXtjBUXGo_(8*7pS*hZ!mAJB4&EKo<7bb! -zz`-Y8y6xy+^t;D@biXnpxc!2nji;?U7P)lV_a5uZ)+@B9xBs5#X+OW&`mWF3p7|1^ -z4vw2W^qVqwa-Y5W^_a4gXC3@BYIKBc<^i9p19#8c`+BQrtzW=y$H%)y-P`!-oO`#+ -z6qx)#qa9pJ|(#d##;1Z`+=(g_d;xSqy8^ -zaN#TaJlbxbmAl%gY|S@!TC!7%t-PX?{ge0WeRaC>>^o25_I?ys>p`7j-+aGq!_e_v -zzgW>AZ}h}gJ$ANz!{_AZJ6eBr*RRao=!Hjn*RVaTJARM$Zr+t0zltwDXxpmI@4maF -zUf9@#l-9LIO$$6buy3mE>KEBo%y^~i<=SQU97+4C*!xYt&h~NMhNF)L*YbU->-0qx -zw548UEc5!VjXHEJpXd2Sm$ybfJb!lkmpyCUzS?X3#GapK_s{e7tdPwEPW?H;nxo5u -zjk9;;>#=72lIs!8`;9ogcyh`9Up_m!e88ID+OPjAXtg7y->|1wOZ?h!&XJWh`#kHh -z_4RFMuMW63pm5O*b!+>Fw{Q8*dpB+dec*UHa>18(z7A~k&B5dF)v{N;xqHL$^b5K6 -zz86^NrP}Ei<45Jb{d%tp3#Y`18r|;nXrFkr&x5iRBmLh0D9@=pkq=J~_;zFDt-~w6 -z`t|i21;Wm@opZR~;$C(7531BS_w1Ud+vopn(I;X-v;2qOsWhQTn_}}yO=`6$s(3)D -z${Ro5cp&EN^4y;^x@*gqoK`!bY1@Lcs(pFA=l)Tzmu}mxWWv@wfen^*=rJq$+3ECW -z-QO9JGACE|h!O?ImOA*``j3VT$lYsr+cxXpcqPIoymPk}S2tIPdt5%O!OO#H6`65m -z;2$G~uK6J9+`dE0B!u9KUS{7Y4TQoy5s%RC#SXHOWvdd}-T`sa=D?f2)-Jx!Ob -z7+(6(@9*|~xyUPpfAIdg+==RyFH|eM<;al4J3qyh`?h}k-tcl8-|Ez0SM=S)Q{7`r -z)E^s`v*L_7Tbefb@RH}m)oZ-FzWH_CDNCl;E|Xp^{-$?WxNYi?+N+Puy7>O5j}v3A -zl=%A9IOj6E++O7T0spx=D{WF3Y!Q|NLrsV>fvxtp0pUUt8&?qicM>cz&+^Ye&5_ -z7C)mus__1eudjUEbb)v8(#u!oKl5SVq1n9Wx9pv##)OfUm0oSDjp#Jsy~Fp`HyhG( -z@$hGVB$SCc(fg%Ykqsi#H{Y2*&2RP>a|SsYb?LpS$(cXiC^qL=jkGUcU)P~>y>hEQ -zxbYz5>^FZT41K$|RxDTccTT)BJ3YNa$FJWhJ!j}^U#=aR-uC$8pNCg@Kks)nb~M@$ -zeYET`|Nc40A1t2DW9`Yao<{@rZaUd#=qve;|6DlnK;)Jdq5W1}`NE??;E{ICw`H$3 -ze)8s_lQw?W^77u_e$4yp+L>h*b^Gk#k5`%ncKTF5o51~8TQgz5I7`^8&lT9Iz^)7e -zAf-$ELghd%@ahT%LbhnB;6UJfmXLtj$S -zj(TG^V2lCTA?A}YLNY)|!h|FYm(Vds7IJt -zJ<3`;YSo8O`(8peF$cDzPO+Yj2lcYT$NXT_BZI-HN0=F1cB}V2{1yx09};F(k2>DL -zsMp~I^*ARiN4<1>_)|9>W`>WtCho~{@kiG6IOi-zJ!L&SKL#pFkQqkT7|1w_b`D}il&?g& -zw^1Hc06ZA@2OzJuf&UijltnrfeR1kEG^6^Nk{`guZ90OER;_id91Z; -zy*y^&?-eh4`}vmeYvn%z`B*N7qw#Ygt9(8YV`y)DWJ;d7CpRj#$FF9aBNjyw)Cs;4 -z0^5-5e>32)gC5-Zbf;f;lqP07sssA~T)%!=OxU^K_f`0d@< -zC0E731N}`a516-Lkpu4~<N#t<9&rdl!ODLitl$_M%ojzcfeA}6jNI4)qMzi}LJ -z{0e!O6mN>P;-U!D^5YYzT}0|l5$)SbzNL$0h9#@K4+pIw}b6_TjJirLS-{6MpaMUb&7^ -z|H7YbQgxgmLg0VWO~F&k__ob=&$Yjoq5Xdgmjx&!%4bWm0@ -z1c~#Gih}>3D8n73&YO6A=$MEWuJX>}p7so~Rl$~pKKvd1wZHy6zcyGNhvS(eOgn@( -zR&6-r{uCPPE53Jp>$u{`2Fqj>_Z>H2BXXhiPiUkKS9!z@$9-soA8gna)coGz3vGT1 -zn_>f&Lm~`3^V}T{fMu~>{2hL8Wy0?@$1PB~4?5l=J2;8LDHR`a*KrO!eC62ZIEA@b -z2ER+QP23atw2JuT!*BF#LrxiJ=PA_q3N_C;?n0)aXq^jKyus&vV7=wIrdz%B-Qg#h -z;vrm{W2%rJ{EDA*V6ejJA2m*dRg8~WK_Rpk2AWgGgdXsXOfebP=BS?v;e10-2w%j5 -zBD+{L)GsSrw&`sBY~{>^zhZr+>m+HjU_I!W;~}KY@jsQpU-ciaQ_KGMME^}O8Q12> -z-x}iQ?~Z}$zYzPa4KYXk(c?&H?8l;$)e1Xy(Nb7@ig}>uuLnJIG}Gqwe@=g`wY&(! -zY$5$HF-g?6h5=r*+{AYnbXh>VYzQTRPP(El?q0sBB+?#8Dw=oJ{XPn*1^JM`SSrugMn!uMHPRza~En -z%#Yy(0%pqsr07udeE877c;)2I04=`r~)0-a&de -zKEv=kRoUT#h2#D4RA0Mqs!yt?-6Pe*?rHaFg5SaFEM}r6Pqo_d5>^Mif12Qg#3@5& -z^t3GLjBmTXGFBFq{)O=) -zJuORm|MTNVdX^W^`=1^^(zCvh-oG(^W%aR36h$$LiTe0)^eX;q6MhgzF#6*YX$Ajz -zjCSi$?i)5D5-y64XA#ou5Bj3_C%5JuS<^$*xK%MY7Qd5g2gJ{V2LqQ+P*4G%*j}2b -zs`^qLZAS{(m)xkETeC&>CVlU6ks4^)#E57`w=D84$SZ2ljb}ZBzs|xpGC_F7E&=V1 -z_;LH!AS)$rq1USzqzTDyGGxWNmo!m5$jGDodRj!aWaQsR{y~Yy?kR$!DGS`0MO!T1 -zk*Om_M9ze4A?t`R10DtZ)iJhUvarQA#dV5B)L^~nFzBC1XWv{*&;O%745u~8=mby|R?FX)UFB_pa}3{vfqpJl)+YL9`AjEG!f -zH-vg`xu-O)seAe2qP^&!;%%qr|^zTj61pd&WuyDjhgz*E`8>qC^8<$f7y(g^{s -zlfcF6L*Oj{f0)3zbsl&*U(cdmtkH)s^x*_#6NWaqJcDC%v#ux(V-XCU@QDmE^e_oJ -zXf@iL1>aMjiE5!n`Vi7)d(Z*+tD~GWhXEgH_C#KcQGXHYlU9I-q=joWpJET=j~y_t -zf&pv9oBHaH_6bJ&4rnjLn8=t^*BD*LcqT)3z=<*P4F;^iIi%+0T9k8M`iVfyTR+yT -zTHL^g0fpQ=BF;k2*jdS28f`iEdChqrD00LS!opYc5MD8uk5vI9(65MU295*Zk2+(X -z-_$W7$|Fxy`}j;kB<0ZyeeUI9i^Qu$ux}m_!$Bt%bZjCu0#71JV7EGgbIpA|GR?UN -zToJR4@|=JkVuV|L{&}+|@c#0arCa+t0LbjX?y}2#ijj@kc06PPg1Ej3-BnQQQj=tDLm1y!{mVrCo!l;iwat$FLkHLpcgksFA;E_`( -z`b@d&Y1qNgNJpTQg9u_@f+8sYvU+*(Xv}lH4&{mWFi~DMBHgcSY?2p#FCA+n>zWtO -z%X$_A{SW_5sOC-u$GSe!dem5x-zU-Xrw>>KsP|-jj{>P -zNKX-ycasoRhl^a1TZKrf0*K#f$I6}!&=%PftAfMfr>}yrq)EMmEs7d)7b`8xk%m;V -z6}$`_Y@={`V+E|991w|Fnp8a~0JyRvii9-KV8o1b;*-0@)*%!zASp5s-ES-I&BaVtHf?EI@t{0UU6fMF2*_BO)*I=mD@F -z=K48=4@S`Fzb0yU3!l*apaH8MyVsP)^8t)9>4-!NaDjeQnDC796gfg{!q{;pEq#$88S{sAL5W^ -z`+8`P_q#)rfiDD5jYD6w;aKCTDg1BPn=_(BWHW1GS!gjoh0(qepu`i#nAu$&e`&1&6hXe%OxHZD?F -z00p4`;z<0pLu6XuvdC25(fYj#*GiFq{Gbt$RiHuHp+OcQ>-QEb(QZ6u&thoS2w2F) -z(3laJ9g9JS^_NB#6r+G|Nu=Flq@K=>bQQ4#ct%FrMPTFrgJ*Zkq-I=ZR^tY&TV__{ -zD)N=p;I*MKzj+@* -z>p+?1GV1G=B4$M7ebA>&U|}(%Ln9J&%Ml8hjf$X*RKDGrKN3E*E8kV0`OBU4UHLu{ -zPeEJZsqMtWdrE2HePH46zg~Jh;!tvKf^p{9Qb*MIxR0Ca7grtXgBoWw?vw|7h{z8O -zE9oV2!20+^@SZm_JyPuzJzB4M@XgUiZ>KElYFLb3nBi%_p9=hHg^dwg3faV#2+e>Z -z?Ng*bg5YIJ0c9F~5gsg5 -zimeGN -zIm(pfZ3YXs2l7ysm$ETiUe)FX+9*rQHuq6aS=y}H4t5!Oo_`qx;MT&+fmlx44gSdUaRb -zygKZs=w4m2?!m=keoK$3H0%&OsHhNqUfN=D(}0zpHpV9gdmOMav5_V8xg+y%t>=|T -z9?RTqdyJsBxf8bb6Tm5eQvkUyLnu8(@|miA6q%}x#j3v+`I90=%twea-5Y6~g24~Y -zNEi>B5Dz;L1Q-O!Hs(59H#F$}Vel4U6~)hhH_ipX8cCi4zi1H*oBlD{P6j{HgX{oL -zRPtUa))RIU_BJUVo~W`t^m4jGuavEqww?EKE|0L8VWa60@>Q!rZRXe?MA-tA9Yz_vDZLK%KlHI@ -z+Z!myN&ey%*bCNAN4k)aZVRY%q=QatN|w6zBH6KweaK9%Ye@@q`@sI?$DWXeJB;4K -z(|Je6Iq{z33tirng;@p;Q%2jSXmr@hhM8|?|*@sx)hu4BTJ1fJ^}Y2u+B78U*U^(v#?{XB!+^%Fah-u1t3 -ze+%O7`WXD(q8=q9J9?A?{O|nT_aP7I@4D=34mbO1flX5OH61pZHZ;U!UvaJH!GXxs -zzDipv?I`KF`^O>({%_ED_<1W3$6*{RMP=pd(AVW$m}uC|_YJ!V=(L+@)|K#yGy2oY -zulzgyG<|7hbCtcl5B!fo>z~@&ny{P7m*%yyx4fpkHT%b{U_{NM^uOJGV&#J?-R! -ztfTBb@w0tL)S=Dqk22~xZ7*#;)5_*kzgc$*%2KW4aqmmM$79~e`$;wC%K!E^{Pnaz -z>31g>Y2}YAe_r|J`h6-5oeQ|7HX6cJJLz%MiSQC@l -zp}n&4zO)mjT@bY{Of=bZwwaE4%AT{$Kl7`fxA89jn$K4GJeAK-`P}pnqP;iHiXq0} -zIj8=t6VF5WjFitsli$WW4TuSS@xBG(?^e^+h;iA?*J9UjojTj9!(~&gb_>#WBfZ{8 -zPhBs|*H4nL=QB;OA3s^9%NnqS0go6kYKj$GI5_$Co~$?EU5S7Gbo?10@;lFvFb$CP -zx}hHP56qPH$|9fqjYd1dT)+z;$T#84jTzH%yJdgc81#}2Sk{1SM>?rrO8$?4KjPhQ -zKyT1v|1VFIa4-7F^dUnIcMVwGD4zq!_AAkz^$Opx*a1YERq`?3z*~EwtXCa;uzbI# -zEN^1ud(V;iEkKXuwfD&SdqIzB)!&M{Wxh&JLYj0h23za^;%ShCM+`n5UXXMu?lJOz -zkudI>gw?N0c)*}{-Jm~UwM8F?r|V?7{}y2v2;IOh=`H%q7}r^nzqMaTI&p^HbTi7w -z@00oDm+j&Wd8V$F_zPc^aP7wyJBq~sL(ftTxlBU;HRR7R;CzF>WP{G6Y!*9;MXoQc -zcycSme56_5X2?Cw(AVBZdaa?C2L`+3$ok|@K_rE3uGGWJfXtscPr@dE>_=ZCJ;Z>c -z0g>$-Co$5MroK%=Ii7}yZo8!3O&TllC2WAdF}EyszSkGO(_#k@ -zlOP|?lM_ZiZUK_cWy}-8p6EB>2?I94d?NjafNYm$%)^-7nx4L8^m7yRP|1Pq?a&JV -zadxwW?@HRlK5Nq+caa;iYLo*-#za4RI -zP9b*W65{u8AzEYI_!+*<{b+&Hm51<4cRcUmaNNA%B?{sHMs0Wl!YUFj@|WmZp$tuo -z&_e9>d0B*@G)#C1kl}5PC@H)}UZIJ6h`Qs6v>#5)^4%P~?TcqH9)fxjif;{Vv+2KZ -zcqzc5|Ab;wl(Z3c{I_YUNHB1R;a`@(28y5WFM?~?-2Iv9EG~c!%r{L}&kHU65v@2IJV}=i -z%Vr|F0%ehaGAru)g6AN}BM3{QuUn4`;(uELk?#jOeDq6c`(tzniFF3fa0xLTb=c>= -z_}^*Vndv>QCq^6OGSZP0*;n?_AJN5qyoh&VZifF-a+ijC{V-GgBxpp&K7vhA9WhW$ -z)G>L2ua^aOHE?2=y_Rx=y2;T-b!tbLQ5t|g<0J$6X+w=9@aY9i{)jGqOMre7a{Q2Q -z6`e&*DIe6Qe9+w4n~a<#fl3LqpeAUBZk5I_;3nbio$_Kl^4R_c@Ki)zJf5rKWe5bm -z6-b@02`T$$&?}BMd3Q4qlKMt -zF%md}^wI#mlp~ZI^0WfKl&?Q(x{p&5=0V1>iVzLZDhA)e`P~X*R@*{L&d>nV?BKf` -z{t3CFKBn$^D|^WvxipG6dN^^o_s=D@ioi+BHrisgY~w7FHv3vwE3B~}ne``QUuau2 -z*tjH70jFrkgM%u#uIM(Nh -zj5mKxp>s+2cWj*-lf)(>w-su0Ete}ACE;4lhFQ&Ho+_*9HTUvwwa|{MSrV?WP$H=H -zEbDa $z`{i0F*DK1FGo4DM);ekVtHHv{vC3%Y$u;&Djo`91Ty?KG^Qq&PtZJUr -zg7qjP_+46Gon7_n!9vTBkuS+nb#Oo%UITA8CMLoKNPr7~xhSxXYr0G^3{tkp^Fv>x -zuK3|vU2*nPf7CpQ7T^0TOjcl#pD2`D!LtRFQan)}!<;-Q^@P5q`scyufCR4iI~B5o -zY5DL(wUHQ#A{WuZ6(|7QyVe}eK=Nxbp!|&rrJ?MrFUAxnFF}{HgEb|Tzf!?BTY!!Y%S?<j}Kpzo^C9 -zq#C2Qr#{i2NYr(c-j&~TUK7{&Fz}1B9q^X^%1`3haHZiYlmtS`SEDYLqWh``>RfIQ -zX?v*69NBETzTts_5o9i{QRe;>wYZbBHZeB#P+Bb81{iJWlhNuC(ifAITy6h~lKV*+)l=AM>2(hNS#mwkm(6_U~8)k2<@H41a($}cq9AhKA#X*2)>2Bns=}?4p*rpTw5bA?llpZ;6rl?meK_&bsbn^&GMdxO^uwg(}bKS%5v&f!ikl+(vH$%(SPgg -zWnRm>D)Jbe_wiAhOvrXIZkSVFfO%R5%ri5=OsNwytux{C7K8NU43NGd&ZQdq`&a%U -zM_I{6%1cSaT~aRpkiM7l4^_{lhRIdv#Wi2t6Xhw=Yn0w%1!0F4dprB{cf3Zm8fMhg -zWJqa0-g0#LjryqLe_LdT7S}b}yQUmMJw%JzT9g#CJRjk-oNkTp??EmDxNh{d>2j1LOVQ2%Xv_*Uu -z)|dCg)KALsiBpbkFeBtX3s*AUJ8;i}dk(Fz{^sDz&3aI430va{d+q!^wX)8^YuFf3V@s~baGZ7 -zG7dzZCU!XUT0sXk`Rr$91lBi~S&&i-cnh)BkF~$cQ2Uq(lSLf!QS*Uv=dKy|g(%|- -z&U)N^$hiJy>?> -zkeT)7ZWpPN5-HM~=RTAfM{Z!jLvfup?~0^nk#P@AaY`tCg^IYVri3!1ngiH)$KZ|$ -zWOYucw`NQ=bv^Sje!+v90cQ -z#9782ZO*il&N6cGlQX4i#%EG9BX@N$Mwxld?BiW0KGJ&NOiBKfM=WhW>-$qd^}zap -z&XlU}^0ULhy{CwC)?obt&Xlary>i;BaC|HKz@12ra3|Ey%0s8NE3G%reCEqL6W*m! -z?>qaOEy+YD{jBwKy4C=CX9rhYo0G)*+TWwby9)B_uisI~x!{tY+G8gJq|LeO>Tw~H -z-yUROQQn=J*@Wg9PYH5faz)PEZX}@}lq8NZLKYU0Bb!=J=rQ8mJGA^OFI(BCc=U>` -zLeMJ2*(wBj#T_l~h9qInfn(eNzoI8Y1K6~TJGc#zBX=k?9UJY8w4JJFa`f0UQse%S -z>}8^!rge$O>`}6D^+V=!%;f%NW^(5I`Oe85wd3t=*C0bK^cL3HL2>EL7UAr$d3+ -zwKq&F5}>gU#7xLhC*)i<-$AesHIe7O@2s~qwfScrIyub95l72rT18zu!}NGY;EAJL -zj--by#98n}-l*pcEksFg@^ll2JEh0Y`qFDp!qs2x50lC+CzY)H@h`!l49Xv8hP3hY -zKWG8CddU&aY(F^(JXh178PX2i2rw2|y&t!o*j+wqV4$MNS>hRJo+aioczuD&rqD<2 -z>$Zw!SLS*s_m$<9v>k5F -zq@_`NZn9ln{g~Fh&S}nqt{Ig98>fC6-Uw?Q$`FGp>=$M&UG(Je;5H`NO_B! -zLFO54F0-Jf8X4XVsJj66LO#F94K3r{0Qr$Q@~&w#G)`WPG42$&I>;RSRsif2pW>VE -z5(Z?doi52`yi=%a>VcQ28=Uo>0`J%}?zq!t^NyiBRz~iylg14|xzkP!8|%dWpWZQK -z^tp+1i4&*v_^G?}_%rjzxf+`_i29`bf)bFuODb~jIxBlpQ1{I;+tqg0Hl;htdPvI@ -z51q6^TWx>Gcvk7S(Gu{bg<~2Pp -z2iYIqVO79~a?8vYj4-@x>9v^spRDEdJjb-Vq|^X&A~Q?t8nuj`_%9)9)s+6XnG5Mt -zyUR;)_2T}%W5`W<;&%RlgdSe(i9e5%eIZZMKX+Lb{Ei25rR9-3ozxY1XONYzZth!F -zKC{I2JRbuvYdsK2Md}2#?ZqDcZfCFc`@c}Lr8AdV#O_d+rTZryzO>pI=MA4GxV-&; -z%g>jyqJr*&X7usplLL9j`gc5hp8v{xuUgF?X%GI1cV8T5V$>}7XCA(KR=~3*oHO*v -z)!g~-ef+HY$-67*kxkU+%>d9;Ir8*Ds>p1=o<7ohVai9!>fS)kJl+HF90UESB;3ap -z!#&MZtigD5Hq1-pyNqlHe`AsU##h`F(*w)Dl;!wK;^)g-U}cwfH%WY>&s6ClQtA`L -zd_2|UeE})T(rmi5!!I8h)^GxBGi%5?-x_NMbHhxzoVn^07dZ;bt~j26Sy{rUNzMy$ -z156LB7bok1^Q`Ml0HvBbed&uz&HWDdmE}YX{6nP$N#cP1OiY%44-PY)6gB&;?Xs&e -zG5f9fhao3#-lA9byT;8A&ybS$-p+EfUn~2bUzb3f{vEG>tM!YJ(pL76{xnapD87~U -zCW(W_T`c>>^HXZve$dA)2qz=C7eg9BSVj1qfd*zmR+taL|4tG=>K@%WXTP3*j>TSH -zb@tMHN6ub`K1Yvt!H0F3@x)J2JVE1qY(^bU)pa;~4y6%FM+@Z88{&(a@RBSy>9DJu -zI?QKua&_TzuFg)`e521ta_73Ada+(hoBW^Y^$=$--1YVS4857^>`m77ves+qi_o(D -z6P@0lBQu>oW8%wOoXFv>(0a!mbw`_BRzdZ_jRlTYSg>KdUMvt=Hi(PA6WNtuq_sy -zVjSKjkilf%k)hi5(9gv70G(b&UUB>${fn%9(_ER&jMh#%X3eK;va+<7bPLP%Me;(+ -z&AcT1*Mr=DWS`#6&?lZM!n?^%TFSO3IoD;%bxp1p>+2?G`^@!~Tu_fj>VhLLMRA2_ -zph~WZ~dpKK9Gi^Ko8_ygqvHt%BcO+XZm?AolXUp -z)M@Hhl1RZ7*Fs)pW#QfRRoZW9p}0TBwSd^@EoH8I(ie8uKf}s`yCflX>hJ2)(D2Ok -zX_Qkce@~xW*5y1#&uld_>J+V|OIysoINK@t4aGQliyy*YtVbuDJU*{S(k^w@S6Ow( -z?$Rk*q+;Nsvu>lh(DGqaJ>Ufo!tCXNVjF&cJrMpbuXw(hV{XH*h{JQlrOl)R94G34 -z^wj?DRSJH~UA|qxeFuJ+1`*(VWuTo{sMp|A8)@e_FI_cc-apmM -zZ}GDgl)LN14zZp!(e)N5Bb1IPzxI@~{O@V5w0d&TbDefRV;`D+%m960lkKjhkK3Z?Pd}*l{D)(X=c9GH*!*WUgzmQlXsV%7d33A -zT!C*HI#1ga16Jj1qgDH>88BQQhrgN$${J=p6S{(?EaesT&aCy_OmiYL-G6?bysq0x -zWi`70L;cTavm_SRD(F5BDq#HonPn{)m6`t9xrph3<6S;V>6p^riq7)Cr@t9{p}Z2+ -z57J0+(s1dq>#U;78dC=|)2fUVl(eOdUhKr3xxTJ;_U3ua -zDoh)2#F=BZ^UD8}euHbr;d)eV(QhxBdAAUcG0TLw;cVYR_-M2-IL)oOdtG^tD(=^$KG^w^g#J`k9OCup3ZW2{R+_U$piFNB0ygy}#R;qF^fu`+!tUjE*PuW?R?&l5gNln(aFubV#-&Bd?&9oI-=Q0f|c -z%$@rfjIG-JmL>EFrLK{8_^MJmdR^BO2NxEOm%_qprOPkUP(J#T?TAsn5hS&oCt+m? -z-^`JANm-XcH16ohM&E-B{%v$|Q9@Wpxwc7Qc;xE$bjC3i#(N4U|)(*_$ZfghQy?7bS -z_xqEaxtZPX%URO`ePX}(oWBq>hkfPKy(Dp0Q|Cd1{{B0}fb$ZWy%m`8eQ=8?h@E3E -zU`U4z;rW|tn2i>3*vSoT`B7(z=QsjP@7VnFxBK|qN4M3O21>`(EKu*oNu7Vu+kQCR -zV9D%M2A(lvPrmOk>d$pkNnXVkUzp4 -z_?G8$-8pwXx#qblJ}Z__2<00-v=qdt!3!e}_MiCFSzq<$o|^eQ*X6QRF6-irz8|;B -z=jOzc@jW7`6E05pCbl2&@M%Ba!WoIt4#9aZ*Ly$(zysgUCO71XJWDJr;k(!@m#1%; -z!?S(Nm)I1C5lG7ucu^LfbL9EF-1@gKz8fNkrLHlT??6he%iJbX9~Jf_k>4@_@;HU} -z_Pn6;et2p}DbOQfU!>iX?@005rAtzZnb}+=U)G0LfIof=fcE=}*J6>O2mX^FTk>od -zX@r6!sd)&Ij3O`g_J@MP1aERwxNq?w)}8gqj7T#ddsGo$pEV4NpF4AY`8docHjQZ_~U= -zzUm2&`@1Xc@`9iq)Jo`jE8F$fAuVDOuq(@q^GL|ydoAh=2Cbh3c%~Wp;5v;#o8g7p -z^5pAX?XG^Jk#&_MiJ7)tz7;VNW9)i5=QG6^)9I)JyjFp0l|DYM<}(`+om=HoV?DbLR4y&wOUi%$%9CEHj2OLbjk0 -zpB($-TV$aTNy?PE_LVK~kOA1>{BO~nEjv;_{gl#q07s^5S?Lei>Vs&b9xvYwl5H`C -za0PPMX_C8A%306jy?ahSzBeWFIguh~2l-v-(6hCa8`@!#&l7!jnZExCJM`RP?DJeM -zbqrmHWOc9h`aZe4*cX|?9U(0#%f1ZwU1n_6_kXvMYmi+detd0nAhc9K>+{z(Qe*CA -zw2^yr^2Az>Ufu7Mb-$kxYPH^dGWnjbT-*PIr}z8OXo@Z)^d0)qG>`1Pp%#(DAyJL(-xAJ=#}?lEfhfub?^8~Q-@kh -zKI8l%x%(TV-K)crWNB^r*L@naE;)29LqbpD&-R4$T^Y(%+P3p@_okJ(f2fDNk^JL5 -zxX(scU}MgxFh+kR{2l<9=Rso4mOK>zbomNYS=I0_^s-#!M}7) -z==`=b5Iop68-}oPTLf^BOCCO6$rzNTX2PIiVM=+LORgV`R1a0_-2LqNv-^}T>^UlVDojJj= -z!PL%^-+jc@hO*zZGr4ZOFc7<5VG7njF~ -z-v+0UzOZwcmd~$C|1>xtXbL{wxs8(II}3sfI>+$4m6Owa1ttdQux9mY4Z-2vgf^ZPekIXzg&5 -zt4xbU$`yg6BSVh3eU}~?dF0i$gKb+6`yD^^tvvjMZC`|498i`kAZ7=y_iaF8lk0Y$ -zEEUOZpT=kr7SZHdX}vix-xqZ_!Fo#|N?jx-YDRw?ZhG18t#<{M`m9=p7Nsr~ncB7DZCQ5x~{ij-5dyMK6Y3O;4E~zfidx_u&^ -zM7P(Ey>{$X>lcAe-(1T7N<6dvwPSBuzu`JlOV@5#H`}UKd}xiVI0;eOf_ -z@%V%+uCFEMb;Mq8yn2v_)Y(RpHi!5djy!LiAYL#I6FCN-Wv-#w9O{oZR>-!yrJ~Vw -zj~0(TH<;t~c8gP&D1To!%bHhlZ^2~cs=?P-@2^;1;8w)etF0?5Qk3|}pIINNC@mPJ -z+D9+sm0SGX4e}Ptw^4Kl-lT@R=w~1QOVTy65AR0)T~5A(l@_3mS8zZu=g%_^k3Ej!UH#)Iq -z~BBR9;cQYK3})5El%BNNH3G)M}qa2l{2mXQMsk?+?JJA*iw15wYD+L=z7*Wt(*tWW@ND=O0MNKZ_?BQDLHfaWUX}Gtqrzr;Y!w{%Gv0e<%l6h*2Rx6RI4<6 -z4~Z7SmN@Rwr2Be1IgBw@qy*<}NziUwEl0DPBaHrgtod8cuRl69OPw-woiciK%GY(8 -zcw~uno6MyYD7CePiWsR|Zn3nSNnIOp;acWCM~t2~#%SWd2p)mGlyrm2>5keW=6inq -z!f@EO`WkIFt+8#(^~js8?dJYov3iO1oi5qg*~pIVEBmF0?YeA>F580a-<&7=O=Q3M -z!aH5E-$C{}J+fnL*eG~eDmD|Br8c=N`gOW~ozC_3R;?yg)8?se8>KPY>S@iSd^e_eKeWapkI`%YxvxqVre>}AMa)<`G);_L1HCgR-ADmg%xJUABhoz30h(71^(DZ|#!ZitN@N -z*|a(9sI%{a<5ee$61WFW7{4yP$@=yAHd_>Gj}BvoUUr0Db_8XAt(Se1HMTm`hJ9?& -zm8XAXyPn6W=P{BeHd40X&DPP?Lj6WW(5sB}D#dg0iw5v%c0S!Ox_GpHM~to>qxYi| -zMZNcJdbPY~(<8qwz1wK+f3Ql>1E$-U3|X2U3qSeOw{v4 -z>v^KdvsllwJ7HmJoVqmfa~W%w>q*h}T-(0369r2gv%IUSjnNg_6U9p$4c;}?MpkR` -zO1cr=_0_*3-0$5`{TS{N@5XAI_jL8U#Q#RK=jARupiT8W{bKH|dy4C;8IxA6V-%=# -z4tGoc8vIM9RA%of?!J;SGNe+Df_mCC_bksVFB-JU>}sbu-H`70yk1=GG^D>){Xwxg -z-Jca0FExC-`hC6hKUDu&*QUKXO{>UmufAiAIekycq4xKS<@LMzbz@3>draPEdisA< -zpVHHRrl@cJ+aU^d|UDM!A*^2hw(dd_^f6nq&IcM&;V@;K3d{?^k -zt!~o&M5fD`BGc)gvvR7Ovc^|!^*eqyM@IYTthu(QxDIhA6dt43+X$CdW_8XD#s^y9u3tbQ^qJB#5|tt -zc{I&?HK9`Pq(14>ydFYV5!x%|2J1TNGtbmhpWr>zLar@@`v!aqw=I3tq~1k*rnX~k -zrSqO`^F4Ww&d;iF#>A#M{MlcOq8E#^^Z2%{d32Veh?K>8N{gQI^r%ejqxG`Rvi)Se -zL-l=TTNd@bmik^teXpm!Z&KgduKM;1wVQJFpf|hfyQ8bVgNd=^d)wcnw!i7BZQs;s -z-Z#5y+Z%d=)m(FWur~P%tN&VN@mOtgYE~sRnVEG{ib?(R4_e`)ZKtuqCp}j9oVtDS -zZ>$he)30Qf!&Echqn1c}bz7nP4v1!5D*H};ja7?Nt!k!rX5Hr_iZt2thpDZ88G5lE -zYIb}+xITN-PM^~{La1LwSk*HT5$dZ-@uVE!hYyky1;xwg42v&r>Z`{gA?nwb)=>nh=ft`e-2;MGg8omE2Q4Z2({EvGtr -z@y>jt-cyrbl5z9dQWuuI+9rEQ$7*@EhsK7Ynxx~!C6^xeyVlpta?MN4b={cUJ4QTK -zv!P^6*BJ53n$Tz=_cs_bN_u5%MaDK{>^q!zY=^bJ=Gl@&o|&GdJfdcl_zl-g$lcbt~X0&dfp@~J3AC6pYG%swqF@2_Ou;p-><}pJ&Xa;7uyvw -zsLLJP<;M4hyz>WKGW1s#j8PJ^y|E -zF6=2kp{sV-=KEy-K+S*Ha+KXyPFG#NPs?*#<9y3WtNwTDd# -z2z9wIdoHhiZQVY4GH3H@lu=P5jRv)!(VbyZuj(HfyG`os{zgsuUthalVNX*}7e%@O -zdb*o>(%sf49s8U;^7`xPmiDAu-aolV9{aI9@-#hNaZkGSJ?Recd>W~KqEvYvt9@}@ -zr1}=VCu(29Hx1wB+Fkfc@Kx0A#V7D>t=*4rAilcV7JS+Gnrjc>TaNEw?ZI^il(*Q| -zoCev-0j>nTfuXCe9nXDjQRRI{weAu4D^?v}TC1JKp7z@J)*auSar2+oWz6dxEB~UAEPJ+L`@}a+BQAU|v7euIN4kK7;P#thGHt_r>Bf>b?u` -z^|SpvrN3=aN{nqu$^hFPDVpuh6pQWd6tgWaCEB($C9d3&xU|j*7blLWyJ&rC;$3y! -zA>I2ijz2lsKf9b1qnKg277X6z@lSK3Sip#jwVc@$5wv-=m4cq&JGt-uAJ+WsCZ6`;=|0 -zPjki8&$lkGmuoM%i)1TZpkKvzUCD6jXp?cZ?V(*K`J^e-Mn%?p`rZb;57x8KpFU~Po9lB+rR1eD4{}ZF{av}1?u{bX?dRn(r!O5PbFQp^rZkT`YAJWr -zMTwQpQHfieaq1fF$ue2yy1nJ4ajIY2SeA*rsPa=|H`ZUK9a?X8ZLW_~pVaTMIsvL<1x%(WaD&~<$Fq)lZoG(Nq=>6y=XROWKa5?8G7Cm;b9@}Y>=Rc97PK}-A -zwe1`3zf3zd_ERlV_A-C8nk(c;Kg*l4ZyaY6Eg}-F(?w*cAL~2aaq4Bdw9AopYHVi1 -zV(YaHll^z`8JZ_{%BRd=rnayBc)`B5G-SSZo7Pc7@+jgB!#5hu@tlC-Gu -zbS*M{iL1!p;PLOX-z>+}sTobK%jXsN^A?uxtM;2+k2h@b=V<2iCvfdIFS%j!KHvB) -z)UrEeOr9S-|FPPP&d)Z~`r|eGyo&4xM>U_5?m_ELB*;{Iab>OsN5kZXsi(XL8(#Ix -zb!uF>e^MyqcMZSq3h6r^9Bl~tSv%`%+`sHImHQpDUny3+J35!}dqE<-okwy>KV>PNB{Jm})a#+CZ^MprF!XuWMrN`uXTuExp5S$myk*Fj3DX*@Nyy3y~t -zBlV*)zw1suhkKtVtckF@^svQx*j~bR5w=7RyPMBbV}HZ-J^FPXpGzEXljcprmg-^m -z@;SnLl(2UQyH5{WricBFu)h*^zaI7gpQpxt$n|plIzKh8JYf4*<3))bjhl}Dt1QiX -zy75WfJ=6FU?tdJQE3dHiYZ{d(nl>Nrw;|1IZ2GnCs!dPhK12Ltw&CLzzXd{t2r>r_qJJ%_%AsXZI*XbbCfztaF1{fvomzBarfnq`lT&T-&fga -zTY-P+DBdH`$M0*jSUDdA{$?E;m?hsHFv%J5CFf*h&1~jcHT1}Nyi3O8dj2SNrbyB% -z*4MRtXwBjcfGG8PQRKVVIwvsKm*%~9l+pL0)f>2l@LR-w-%0DOfrovec3%`2YP-Xi -zV%zKdVw8F6=}}QUZ^*ri!OR}Z1M+#|o8lD>mn|$UssYk_J>rp+oP>p(y -z(CO9mk^2?i#!*iae^QSRNxo5=f5Yd^q219heWBe^d)`9}ClUTlKq+99WG{Nnz(!Y; -z?q9F_)7)E}Mb2X9EQh(mQgG3f;oklQ#aT-{vCr8Hsyth=y-u%nK}D#(!C;?V -zQU|-F3p{PvZQVPiIZ8mxZJVcv{2P@&sNS~w+T=c1fvfLM*lkLuD@l_(oHts3-A8I@ -z?~c9KkW_XTyX6W;PMefE@<*hW^+=70mGgq!HS}liVP{OvHleHC%sg(rZ~fs1%6&-I -zM&B`&JHH|(M-k<7l_&f4t~UsYMEO~>6WlUPcuVLb3bWj -zk>;&7xgO81%x7nPg>~sDvnx<3LzX`;HC$wUrt*#UkcKgRG`zD*LlYWC_tEfy9u41M -z_aWwcG<;Mq{Q*iZ)isPJjp-Z>pVZSVC(YBOd86%Qq132F*XRzc{AQO%1Nvz6LYGEE -z(WrkPjb852XumQ**XPalvu)O@mzGay@8}x!C)Ln%H2RyKW(8?JB+Z*`^0X^XJ!KfS -zT59%1W#}m;iD#Lg`MdUS)J}uEYgQB|X52Wg!NLB3Iyhq1q&+DzN6ZULCRI8QwTDv3 -zF;9+ldlKTxtDUFDTDLH228XPctKSC4?(wX#%cqd*qAmBYe#|yz%c-%cTN2e3#?iF( -zO5@e5)4UT2n?TqqV>)34de~IL+=LYyrx5liVW-B<;Cem2MAdJ+(KRW@?7ERu*Kd*c -zU-oKw-#w79obk#$*?;ZNNuR67+^WaqC#>j-***T$*d=<*J$lTtg!{W<_KcV7nw6CD -zu<=p!^5I*p`-<_c#h0jVFrFG)s@Jegui;nLPz??2W6S3Sxu-2pfKH8lMlYj6FXJXo -zL~mwnYjmZ#nzm$HkLd3t$ouQX)j!v-LeI}QH+(8xGyg+AUY7hQD|W3EZP -zz!(~1X$hY^X-TSzR!_v-id{!=!rP+zisU(rMTQkSzZhJ=NkwW_y>og@*YYIz>4OSqycg14>QMG=gkrJg06Y(${%63>k7R~H#*X~Sg0)L9Ppuvo*#u|_|Q -zL`#pvTXc!{A@RN*i9gpR-f^D9rQ7f8BT>qW*Vp_%T_|V3wbVw=dD6coq0(99tjB*s -zabU5ss$UePPD%5Y78N)Pr$lO>m}Tfo&ZnxLDY`PN+IgAxCFiE1=x78t!S@{!-DF?3cE0FR~rmS!ClGh|j~(JQq93Q}6sllWu?#JH%Lg70yr0 -z(v{CNC&Vr2CN&G4)`d=KGle|4J684QqGZlY7kmD;eHwgN6yf=Jdjb5TXb7d8DjISO -zDCtCmE4I<)bAA#d-2$YERpapKt>u%xCkR_5q^}sCPxo!YcOqgwZCU6vse>XvDjx2+ -z;6+z)n#)#gb`7uI;u+4B*_Bl7D!yI!yFC}ZIDz=IYVQB)1)g-SCKR6<>!?mIUaemZ -zMX!m)Lyy7X7@H;5=j?6WJ;Sc|o6Y+8ir?v?&|$z!y+><8Q!Tj+exJk?Y3VuacmG1ZgzqLgrrF5v-P+9&24 -zkygTc485`ns~_!=Rfwz`ZD~1sobznA4D^pr^Ul{lbM?<#_0QY%&jtGD?fU0J{qu{# -zGW}`&b7-ver|6&e=&@hw;s2$7zN&`|(PKyIpP%dDR{irc{d0|;X085d(?f>ppBwe` -zztle;*FS%yf2MimzunFx?V%D{FX`5<-orhKZVFlR^?PiKGmRD5zV-v`IbX6y -z-;37wq~+f9CyM&$S-aGvX(xpJYJI0#?pT*7ZRBHEmfk`cjdS-1DZ&QWd -ztLhFnkdODj^s-Fq6XbqEEm-~`Xf3DZ=2K4)IZ9iMz -zcFQ%%cI(&Ek_OW9{!dqp^7I=0yA~Uy*Qexr>hr@b7RpoEr#@THs?PxGbBA7^L+8}z -z5cRo3ug}YReFmIWpF?_m2K1@V9dh>Xs?SlqKI)p@7MlJe_seJBQzLm_M*T>h0jg}( -zPz%N2?|YB&=3$?EY{JgI_t=D;-S^m~YO;Qhk@}oII6)2FWAY8gzI{;MV^`{BaRNy0 -zzV{f$;vjhR7C+`jI><`_ymMdG+gik8R!A_a56y{X+NHQ$6)N_a58Y -zQ@``?u@}4Qw@a@dvoO}_9<}_}rcH@Sm0{_6nAAdsSyBi7+m;@v_uUV^#}Yr>(xE(} -zUvEnr@%OfLfBNnsz3*PU>+E~+;$1>rr1#w+{bVcmKdbLv+)uV||2}|nh<*XX+acUv};XHlQ}EIz9~8uhtTuTR-I^(mu1ck1;i*6X94Ri84wK3bpp -z+(~_&>Z;GvdVLbt?BSgFqV!_cgBMLvGW@RU?Al!oYoB}8oc`ik`^_)z@{PyUcSxF^ -zWi5W*%v|L>@a(H=yJu#vyz1`=4tg({In>U9+iwA@rEM@*Las -z?pi}e#rXHg7ZiMI@2!=4E;8>wN%iHkpOE*HXTR*p`wgN0()0dn*EehTI=@}J*I_EZ -zMoZ-$HPjB{Jbtw9kFC|;HPPP==zf=BjF!sU%D%4eTPVblh8Rv?_VGSV_YPcU?JWrx -zq|Q&cFf}(JF7?)g_|)4H5>gi=B&Ob;U`<_^U`tJN&8h8vuW4xNoOQP(*hxQ}^utI$ -zg7nFxzlih~lm3#k(x;@F%FV8MwYRPtt<6hpbiGjfsW#6b-j)r@u^_raiFkLd~cN^(O{J%`_O)D-6Ysi`-STTOcdotl+JuN>=dlE%bCTf25RB -zbL`o{sa}8W!uY`WMe!Bme;!{szUJ_=gEj9HwWr2b)h6fgp5jdp)Xw5rOXlG_8?7>* -z+0|HEw=RS4QtVnM^^+$bFRZ&p*LY{`3+vuGmZS~UUMt8LwAj11_7iOc`=WAn@6U=v -zyN0aFiSh}dH_ES~}IYny1B<-MTJ -zVmm$f!u2sW*0|RO=!>?xq3c;g3$w#K*~}Yh9oDou5AVih^KRT6-i^C~cjM-)|Hyh- -zo!pJ`D5lW6eQk5rhu#F7N_gMrLJS8<)(ZJf(H$LrU3`()}+iru8BuA9?Fe6zH~Q@SNK -zhb)mDa0d1Wt=k5*>qF^oQl?Mu%ANfj@z1XhIOpE5V~?V}w7$YQF?O-{V4Z=M8t!>{ -zk6AO*`_iVEv))*5;0$%R=k+~ru9snq6fw8yYk2ui@@mgJdlq=MI7dv$_56uk0jCk` -z8$Ivtk*%=A^+BE7--#=idp&XGq1KIJ|JaZ$TlH1B`)HAM?64~JRkTwz?X;D4+D1F2 -zlzwFGUmtoiv%8%_trRCiBYImer8LcJC#-La%CHofu1_oFSbtLAeUG2}?xOxCs=lAP -z(>X^el9q4ou+BVBtNYPv&N*7$UmDWu)^qf_9~~B-qn8Z3|2(}4>*X$MjII9Kn^Jl; -z+2suFyhp`$*~%WW)f4Aw)q++P=V;Z^rB%Z@TD73TOXp}M!&=VKiZLU?!@J5k)<2+K -zjBRJO|623pkba@P=G}cP_3n8(_4j{d{pUG4_4kJ?_0>5#_4lWFm4>rhP=@sXh0bb=s9(e~wOjoar~DO-Z}H_bL8Cu9sZjyTcONYg)rzQ}6!L+_nyDMnjE!ix#aX -z>zZ8KrR$8YD@KAI-?h3g$NKjMlbT}=-Mh0BW+yBiWm0c82g)juD%;o!u{6w -zJ$~kN-hZ#ouFSr5pMTP~`Z?EkjMf2X!K6HEMfNidb^c09+QP~E*`+u0+4l(JA?IMa2N=KZe6 -zr!>mGkYg=uG^uKI$iA{&cV?g6uDi2)+qEJ(jV-S$%ObU}tkv09?mxHg-8H^)|5VSF -z`+F@tp)sNChb+Bh$}CU9{=PPrdtssX?n9@La~emOZz!7@ZO=-}Nt+UI{>s+d9u<7U -z>CXKM=Tmp=pUp=y}wM3xuuT=oTIf`)=}FU`IepC -z?1)x7gSD*TS38$@p4%UxJzplz)O={en4MXby?4KK_mvr0f!(e3AKd5~qxI)o$^DOw -z(MI8p;=LUGThq+!@=cy$^vd5iF1604H*-qO-f(y5UmO2K+uQY%Yrpdzt^cV1jpx_@ -zi;n95pH%g^o)gzxA@~2~x^IH*^r-Q+FGl^)EPh{GXmmgO8!jDIN5IY4KAF|U6jm2g -zSzXNaePq2n5Snr1w;M*T@V3qMg+`?5gv(h+em~*_-*}Pd`&on~5!RbF%{!N{%sydr -ztn&kW`%iSOpzCV(~teM*g> -z)IgUuei_cUQ!YUpgC3sdeWI&msbMH>nwPKs>not{usPNmwAt39jT{x~*C?(3PZ}#M -zG;D#e1;Q2xTOe$Kum!>v2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v -z2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v2wNa*fv^R_76@A)Y=N)^ -z!WIZyAZ&rK1;Q2xTOe$Kum!>v2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$K -zum!>v2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v2wNa*fv^R_76@A) -zY=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2x -zTOe$Kum!>v2wNa*fv^R_76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v2wNa*fv^R_ -z76@A)Y=N)^!WIZyAZ&rK1;Q2xTOe$Kum!>v_`hg@Ja1=atbL`}WD#yLuQM}Vgw1A@~(aaNQ63 -zg9;t=mlFeNKVcBY&XvMP`MI~yH?iUIpHdiMN)A=rCR<)uEbkk|KoGwjA{6-Rq<4eH -zN20_%2ZxG0XhGJ)xR1btV2Kzi2ErhqQANx_ldwSW;1JSX&|Bpi^;etB0^jW0v^NGkAsMyeybGmEBrYI(yB=qRT2H5NXF-O -z21{s1T11FJ%f+`z!C>=+otb&@ote|Le9=MiGjL}@jnITL5b>XOW{#u(KgE4r!aevg -zX}`Tr+T=kI?7$vngGKCTotc5)5CZ=ew1Y3WT-cxnMuO;+dfJ3VQ^DX#y`43Np&{Y`yaET|bx_eugT>UR#ZLbx@7Ix?nH|*mUfjiS -zH{^jOp))h}qs~kl*kK`Ii(tl}7%>ingSrjI4_VE}IOPW-f_gmyd4Hh%|FBj9y%Y_@)jXP4-8CmgCF0#~tgM|h@nxwu&}531zgdSLVF1Ul)9 -z$oiH$)vhqNmlYY=!_-K@HSH5j4YgD1l8-3c?U2PLTh7us}R$YLtj9R75Zl6?B7g%4wG6LO_-Y7Av-b4*Vy; -zjz0i)8!qJHKOw`05C(yvxDZIf1>uVm4(>Izf0VG}8gVsnN1#h2;Gc$n2E2tn2jLL) -zv(pb4N<}R7hzA?kVH$ag$m0XyV~nxUN6V6gk@-q6zUI*n4{~3-%LxM&G_XJ{#Dfj& -zkPIV197z&0$lJl#a0b2wqu$|c;#Db&xI=*?@icys@Q7$O(7*nIPK@cI5Qgmf8{nFTXp -zC+r0=0GB+i#2*JS6rN_RBafT>8DgbaBv#QM7+bN+Ovc7(Farcl7*F`oUzkJ=X?%lL -z3e_#=lt;weTMa_lD8!g@hFTE#MHKg#2R8sREDJV4ju07_GX6q52(W-rT`rR0s@s@y -z8JF#HY&I}25to&05}C}Cu|6R#a6=386yY-$l1BF7X!--)RHQJz6Fvjtk!6E9#OFXd -z^X@@?-pUteVwk^~7gY!SMuwHq=PmR#>TE2hu4VAZD#n}6%nxLLd@7&54Mk82CBQn* -z27%O}w82W+WF>8~l6FAeBg_ZU*k)tCFoFgpChOT5az?uyk6+tP8Xl(Eo=0@z{_(z0rLVbX~3dS1ZBp|il>|u#CF{d%l(C>s~9C^521T$q?7$?(^$40yz9593H9GD4t;D!aHwJ?r) -zadWvAxZ4?5ywV>}{{R)%vbOX?G1%oAjrKI=#t1E!@<(%j+UcV(17<=pX|@X!YfKY! -zpGnk1ITS%Dl)y#^z$VxXZt7iyD>`Y1{VPO3*59y#He^kbim4U+LE_fniWQ}b_ulY7NQjQ54am~H^B_rUBuJB5@N(=)+VR% -zFT=HS9l-T*EpP+$H8){1s$#@Q`lIj37*PZ^2V)!I%VZq*q)hU-L0g?Kwvz7%{88q= -z7+Ks;S%nV@Be;tm6w^QyKgitkps;}v957D0GHxvCSw9|fFBbO6i-iq5T;~w)fh?E- -z<)kfvR7jTjRs(K1ZYh+&$CP;z-iH%#97Mr`;t1|rupOGA2C5(cn_(lALn-)RCb+=@ -z8VG1b&LL=l?NAQ^*aRD)97>@CiogdC!7|8$#jps1NkS~Z%>^&yfCsW*Cd`0o;D%(d -zgAL*#7BocJ -z`KB=z3FNtDIWw8d)?x^7_d~Xqe#Ly*0tUhcfn1+Wy9ynMp8*PC1HnjI6^ck(0=b0c -zft$2$!dsqY{Nj2UWD%A{SSw*W2gQiJ&^$(Q3!ZVz4#l0rbsj8$MUW2#2GuQFB4zPh8k|7mF!cLw!#^L%v$T3}jk9!5b8>Ybwm5u{T -z0BMJJW+uZCIZu`|J~7_bKrKuQm_#FP3+q5Pem8Dl1Y=DjHsv|mDA$aRB=JYm9EEh+ -zY#bbAJURg#V6l*1Z5ktP|J@K#^q&{{p8p_P9Qfw|v1o0-ag6!LDR-aKDEj?`D&)#l -zq{#Id$n$5WLu8%)m^CAA71TgIyzw=0$jyTQw7?1I;2E%)Im|MUF#__y2liyr$uQg; -z$b<5<&ddiGOO}CeD&>8`y0w&gL94uGJ=%G82KnCyaU9Q -zJfq>}UC6zcNj}oGKr5V(phE}jo2R%p8_|b#o3Vzuiv5%caMZF=oPhV?Bpip0Kr;3v -zSBlLLkk?mWrBBfH8uqJx6AT)en*{A6eDuXzDSvcl=3Cfx2J?#;#4|kZKPYb`b=72A -zo-3OiiZ~?OocmM7HL|7}$Fqi8&POdgi!oPcJw~75+FrqW?q#-d3YaI57gv%b;to;1 -zJzpICkhtG4^pj4Nb3gUZg%d973hE}-5Xf4Dta9{nTevrHn;oA#V7l#MHbqD4Mmi4J$OtS1COi1&i+ -z-^L5~-1|)Lqss~S7*2wC_shs5xUFy;j><4GM7+iR(>V5_T9J1|%6#LgNsfi|iQBN} -z0(7{YJh+D_KX39(Yfp) -zD@zz#SgS#c9C~<`?I5jXFl`1k^tGe1eEOE$-PFd>*OGZg;W@>KD_U-GbE~%q_MAJY -z>yf*#)1A2B!7uu|6$K3a10n(p#?wj!L~9R=7%&*hl?X5x4~Ynfjv`zVDdX%B5gtXJIWT$e9F -zHtiQ|5n?1ZG-8wI=~K;*5eJt0X}7!SPxnyXz32iKt~;Q7DQ(NON*)a?5DVkbXPkrm -zH{4`gcOLCe+Lrrh_hr-xSFOn8AuzwNEoYAbfTX8!;a9$7(tXyzu2K25I_x}fv6%5}8f*3)m#q=yggpky -zup66|;{FX+(C5<3oQ-hZ@E5klAm$RrslQ_fSmhJqvlbzi!0i1z4%4Q&kOwD5Gqhd9 -zviHzX)(1TIK>6jJnXdbU*ja*-&jo`smb1@{&gIYnt&qz)x^^P<0|)ctEq{&>9}hGL -z-*dB-Eb?Z?X!Gh7Wf?+Mz0wvS+}Fha@u*!STG -zcm96G-GRUA@jqfGpW8y(c$ts!D)VPEg%AD5zY+{?nl7!#vn+YbumPJ1qJ;LngnRk3 -z$V(N${PhcMy2j$SrM1~=ovLg^b%RN9a9E4dM;1JFYT_bl)nGA=~nnz@UzIJ2T~yg5`y1G8^2NB5w!j -z=y&Y+$`hvVa5{s3BJ4P9KWG#OTO!=Y_e8iCqr)P|1A{b5%bMV~t1zut6ZEk@DTz0r -zdVjGV1|L+!Kp5*&gbT*Oc(|817i>cI0?OuiM~G9*1ITn6Ix|1r-|&}?Pq;@J16$uz -z+#^||c%M?-S-4m&)5?8xF>WOJiVE_@Ch~2B&4YuP`M5<-qbK-A7~Go(I|8=YVCI4i -z%-4_uGhq=-121I34DdiMECvhnRV*~qZrj1fdCY1kl;<*pi$PFn68&H2ECz-_99#fK -zhy(=$w3Nq)r>Ji%^S(IAGcNgkPy$6zPCpDlGqgenhzqa{YljJN8DyFlR$< -zr9TK{u*xCL!y*cJl8-QeQG88nF{kMHPb-ibcLhNq*U|?If?3GE4g~8(%lnKe_#MRMf(L(|jQfZ_WOzgjM#ckQ -zD`Gj^OSpl4G6;slSa87z7z`2Mgkhk-KyZK^EFd5jd6CE|$A-7!4&r(t?gh9DaDVa$ -z=XbahNE-zgLmXTNCb$GH1Qq>Sv7H8M(Cq>8{2aEze-$Q)HLJ;2MEyw{3&}7J{)@2b -z#Yy5r&8M#A+O@|KV{Uix7M={7+LWjreFK)TJ?WfATj@T7!vACa#g?h5jL -zgu9kJ6JZ5;?C5$w=`N(KVy=A|kBD3dAg6%vUl4W>e+2%;uo*r^-b2XJkhK{;h79DD -z!b7Cr#D3{!*a*$A9qOS9Y9Ii)$T4D{fw)DuKG+1CVI!14DL9FDzy!Dq#)AvS!f+S? -zn?7ZJ@h6E~$O9jgg9F@<1v^P&;VdKzav=|VUehiN=}3Z5-u!6u(EARbD2CM$={%$F^=|9ml0y!ZD% -z4OeFbgVD_I8esj10~be#ztA49-Ste+RwslNe|Bn9u!!|q36w%P_*e_JfQVWplmqlr -z$bvlBg#KzWXZWCj1yUgg1m`6l+$@+0GhiCH!3fDP4!(NTB7VDMoUrr$hRRwZ3f`xW -zon)^47^1nhz#xc)Gla$A#sjB4LGe?@-Rbm$D_Mg=0KXdo`18PyzXex(i@l9!=shC% -zbR1n|KHFoF#FIY{#8u=&hddY+!wif24qdS&D8$vIn*m;#23nZ6S>gv5ToBCMj-55! -zy^zd%P|f6v#jPSd$O}T{W3u{Xb_nq{ddv5uc7Oe^;B?M6BVQC^i-GeC``5t*KVx1% -sZZ6MQF}FRp^6j>d=2YSTqxQgsC2u>;rEKMMw}t=07Wltrfmi1Ke|L0MnE(I) - -literal 0 -HcmV?d00001 - -diff --git a/target/linux/ipq807x/base-files/lib/upgrade/platform.sh b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -new file mode 100755 -index 0000000000..2c6b682178 ---- /dev/null -+++ b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -@@ -0,0 +1,73 @@ -+. /lib/functions/system.sh -+ -+qca_do_upgrade() { -+ local tar_file="$1" -+ -+ local board_dir=$(tar tf $tar_file | grep -m 1 '^sysupgrade-.*/$') -+ board_dir=${board_dir%/} -+ local dev=$(find_mtd_chardev "0:HLOS") -+ -+ tar Oxf $tar_file ${board_dir}/kernel | mtd write - ${dev} -+ -+ if [ -n "$UPGRADE_BACKUP" ]; then -+ tar Oxf $tar_file ${board_dir}/root | mtd -j "$UPGRADE_BACKUP" write - rootfs -+ else -+ tar Oxf $tar_file ${board_dir}/root | mtd write - rootfs -+ fi -+} -+ -+platform_check_image() { -+ local magic_long="$(get_magic_long "$1")" -+ board=$(board_name) -+ case $board in -+ cig,wf188|\ -+ cig,wf188n|\ -+ cig,wf194c|\ -+ cig,wf194c4|\ -+ wallys,dr6018|\ -+ edgecore,eap101|\ -+ edgecore,eap102|\ -+ edgecore,eap106|\ -+ tplink,ex227|\ -+ tplink,ex447|\ -+ qcom,ipq6018-cp01|\ -+ qcom,ipq807x-hk01|\ -+ qcom,ipq807x-hk14|\ -+ qcom,ipq5018-mp03.3) -+ [ "$magic_long" = "73797375" ] && return 0 -+ ;; -+ esac -+ return 1 -+} -+ -+platform_do_upgrade() { -+ CI_UBIPART="rootfs" -+ CI_ROOTPART="ubi_rootfs" -+ CI_IPQ807X=1 -+ -+ board=$(board_name) -+ case $board in -+ cig,wf188) -+ qca_do_upgrade $1 -+ ;; -+ cig,wf188n|\ -+ cig,wf194c|\ -+ cig,wf194c4|\ -+ qcom,ipq6018-cp01|\ -+ qcom,ipq807x-hk01|\ -+ qcom,ipq807x-hk14|\ -+ qcom,ipq5018-mp03.3|\ -+ wallys,dr6018|\ -+ tplink,ex447|\ -+ tplink,ex227) -+ nand_upgrade_tar "$1" -+ ;; -+ edgecore,eap106|\ -+ edgecore,eap102|\ -+ edgecore,eap101) -+ CI_UBIPART="rootfs1" -+ [ "$(find_mtd_chardev rootfs)" ] && CI_UBIPART="rootfs" -+ nand_upgrade_tar "$1" -+ ;; -+ esac -+} -diff --git a/target/linux/ipq807x/config-4.4 b/target/linux/ipq807x/config-4.4 -new file mode 100644 -index 0000000000..1b4b95b968 ---- /dev/null -+++ b/target/linux/ipq807x/config-4.4 -@@ -0,0 +1,828 @@ -+# CONFIG_AHCI_IPQ is not set -+CONFIG_ALIGNMENT_TRAP=y -+# CONFIG_ALLOW_DEV_COREDUMP is not set -+# CONFIG_AMBA_PL08X is not set -+# CONFIG_APM_EMULATION is not set -+# CONFIG_APQ_GCC_8084 is not set -+# CONFIG_APQ_MMCC_8084 is not set -+# CONFIG_AR8216_PHY is not set -+CONFIG_ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE=y -+CONFIG_ARCH_HAS_ELF_RANDOMIZE=y -+CONFIG_ARCH_HAS_GCOV_PROFILE_ALL=y -+CONFIG_ARCH_HAS_SG_CHAIN=y -+CONFIG_ARCH_HAS_TICK_BROADCAST=y -+CONFIG_ARCH_HAVE_CUSTOM_GPIO_H=y -+# CONFIG_ARCH_IPQ40XX is not set -+# CONFIG_ARCH_IPQ806x is not set -+# CONFIG_ARCH_IPQ807x is not set -+# CONFIG_ARCH_IPQ6018 is not set -+# CONFIG_ARCH_IPQ5018 is not set -+CONFIG_ARCH_MIGHT_HAVE_PC_PARPORT=y -+# CONFIG_ARCH_MSM8960 is not set -+# CONFIG_ARCH_MSM8974 is not set -+CONFIG_ARCH_MSM8X60=y -+CONFIG_ARCH_MULTIPLATFORM=y -+# CONFIG_ARCH_MULTI_CPU_AUTO is not set -+CONFIG_ARCH_MULTI_V6_V7=y -+CONFIG_ARCH_MULTI_V7=y -+CONFIG_ARCH_NR_GPIO=0 -+CONFIG_ARCH_QCOM=y -+CONFIG_QSEECOM=m -+# CONFIG_ARCH_SELECT_MEMORY_MODEL is not set -+# CONFIG_ARCH_SPARSEMEM_DEFAULT is not set -+CONFIG_ARCH_SUPPORTS_ATOMIC_RMW=y -+CONFIG_ARCH_SUPPORTS_BIG_ENDIAN=y -+CONFIG_ARCH_SUPPORTS_UPROBES=y -+CONFIG_ARCH_SUSPEND_POSSIBLE=y -+CONFIG_ARCH_USE_BUILTIN_BSWAP=y -+CONFIG_ARCH_USE_CMPXCHG_LOCKREF=y -+CONFIG_ARCH_WANT_GENERAL_HUGETLB=y -+CONFIG_ARCH_WANT_IPC_PARSE_VERSION=y -+CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y -+CONFIG_ARM=y -+CONFIG_ARM_AMBA=y -+CONFIG_ARM_ARCH_TIMER=y -+CONFIG_ARM_ARCH_TIMER_EVTSTREAM=y -+# CONFIG_ARM_ATAG_DTB_COMPAT is not set -+CONFIG_ARM_CCI=y -+CONFIG_ARM_CCI400_COMMON=y -+CONFIG_ARM_CCI400_PMU=y -+CONFIG_ARM_CCI_PMU=y -+CONFIG_ARM_CPU_SUSPEND=y -+CONFIG_ARM_GIC=y -+CONFIG_ARM_HAS_SG_CHAIN=y -+# CONFIG_ARM_HIGHBANK_CPUIDLE is not set -+CONFIG_ARM_CPUIDLE=y -+CONFIG_ARM_L1_CACHE_SHIFT=6 -+CONFIG_ARM_L1_CACHE_SHIFT_6=y -+# CONFIG_ARM_LPAE is not set -+CONFIG_ARM_MODULE_PLTS=y -+CONFIG_ARM_PATCH_PHYS_VIRT=y -+CONFIG_ARM_PMU=y -+CONFIG_ARM_PSCI=y -+CONFIG_ARM_PSCI_FW=y -+CONFIG_ARM_QCOM_CPUFREQ=y -+# CONFIG_ARM_SMMU is not set -+# CONFIG_ARM_SP805_WATCHDOG is not set -+CONFIG_ARM_THUMB=y -+# CONFIG_ARM_THUMBEE is not set -+CONFIG_ARM_UNWIND=y -+CONFIG_ARM_VIRT_EXT=y -+CONFIG_AT803X_PHY=y -+# CONFIG_ATA is not set -+CONFIG_BLK_DEV_LOOP=y -+CONFIG_BLK_DEV_NVME=y -+CONFIG_BLK_DEV_RAM=y -+CONFIG_BLK_DEV_RAM_COUNT=16 -+CONFIG_BLK_DEV_RAM_SIZE=4096 -+# CONFIG_VIRTIO_BLK is not set -+# CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set -+CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0 -+CONFIG_BOUNCE=y -+CONFIG_BUILD_BIN2C=y -+# CONFIG_CNSS_QCN9000 is not set -+# CONFIG_CNSS2 is not set -+# CONFIG_CNSS2_GENL is not set -+# CONFIG_CNSS2_DEBUG is not set -+# CONFIG_CNSS2_PM is not set -+# CONFIG_CNSS2_PCI_DRIVER is not set -+# CONFIG_CNSS2_CALIBRATION_SUPPORT is not set -+# CONFIG_CNSS2_SMMU is not set -+# CONFIG_CNSS2_RAMDUMP is not set -+# CONFIG_CACHE_L2X0 is not set -+CONFIG_CC_OPTIMIZE_FOR_SIZE=y -+CONFIG_CC_STACKPROTECTOR=y -+# CONFIG_CC_STACKPROTECTOR_NONE is not set -+CONFIG_CC_STACKPROTECTOR_REGULAR=y -+# CONFIG_CHARGER_QCOM_SMBB is not set -+CONFIG_CLEANCACHE=y -+CONFIG_CLKDEV_LOOKUP=y -+CONFIG_CLKSRC_OF=y -+CONFIG_CLKSRC_PROBE=y -+CONFIG_CLKSRC_QCOM=y -+CONFIG_CLONE_BACKWARDS=y -+CONFIG_COMMON_CLK=y -+CONFIG_COMMON_CLK_QCOM=y -+CONFIG_CONFIGFS_FS=y -+CONFIG_COREDUMP=y -+# CONFIG_CORESIGHT is not set -+# CONFIG_CORESIGHT_CSR is not set -+# CONFIG_CORESIGHT_CTI is not set -+# NFIG_CORESIGHT_EVENT is not set -+# CONFIG_CORESIGHT_HWEVENT is not set -+# CONFIG_CORESIGHT_LINKS_AND_SINKS is not set -+# CONFIG_CORESIGHT_LINK_AND_SINK_TMC is not set -+# CONFIG_CORESIGHT_QCOM_REPLICATOR is not set -+# CONFIG_CORESIGHT_QPDI is not set -+# CONFIG_CORESIGHT_SINK_ETBV10 is not set -+# CONFIG_CORESIGHT_SINK_TPIU is not set -+# CONFIG_CORESIGHT_SOURCE_DUMMY is not set -+# CONFIG_CORESIGHT_SOURCE_ETM3X is not set -+# CONFIG_CORESIGHT_SOURCE_ETM4X is not set -+# CONFIG_CORESIGHT_REMOTE_ETM is not set -+# CONFIG_CORESIGHT_STM is not set -+# CONFIG_CORESIGHT_TPDA is not set -+# CONFIG_CORESIGHT_TPDM is not set -+# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set -+# CONFIG_CORESIGHT_STREAM is not set -+CONFIG_CPUFREQ_DT=y -+CONFIG_CPUFREQ_DT_PLATDEV=y -+CONFIG_CPU_32v6K=y -+CONFIG_CPU_32v7=y -+CONFIG_CPU_ABRT_EV7=y -+# CONFIG_CPU_BIG_ENDIAN is not set -+# CONFIG_CPU_BPREDICT_DISABLE is not set -+CONFIG_CPU_CACHE_V7=y -+CONFIG_CPU_CACHE_VIPT=y -+CONFIG_CPU_COPY_V6=y -+CONFIG_CPU_CP15=y -+CONFIG_CPU_CP15_MMU=y -+# CONFIG_CPU_SW_DOMAIN_PAN is not set -+CONFIG_CPU_FREQ=y -+CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y -+CONFIG_CPU_FREQ_GOV_COMMON=y -+CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y -+CONFIG_CPU_FREQ_GOV_ONDEMAND=y -+CONFIG_CPU_FREQ_GOV_PERFORMANCE=y -+CONFIG_CPU_FREQ_GOV_POWERSAVE=y -+CONFIG_CPU_FREQ_GOV_USERSPACE=y -+CONFIG_CPU_FREQ_STAT=y -+CONFIG_CPU_HAS_ASID=y -+# CONFIG_CPU_ICACHE_DISABLE is not set -+CONFIG_CPU_IDLE=y -+CONFIG_CPU_IDLE_GOV_LADDER=y -+CONFIG_CPU_IDLE_GOV_MENU=y -+CONFIG_CPU_PABRT_V7=y -+CONFIG_CPU_PM=y -+CONFIG_CPU_RMAP=y -+# CONFIG_CPU_THERMAL is not set -+CONFIG_CPU_TLB_V7=y -+CONFIG_CPU_V7=y -+CONFIG_CRC16=y -+# CONFIG_CRC32_SARWATE is not set -+CONFIG_CRC32_SLICEBY8=y -+CONFIG_CROSS_MEMORY_ATTACH=y -+# CONFIG_CRYPTO_DEV_QCOM_MSM_QCE is not set -+# CONFIG_CRYPTO_DEV_OTA_CRYPTO is not set -+# CONFIG_FIPS_ENABLE is not set -+CONFIG_CRYPTO_AEAD=y -+CONFIG_CRYPTO_AEAD2=y -+CONFIG_CRYPTO_BLKCIPHER=y -+CONFIG_CRYPTO_BLKCIPHER2=y -+CONFIG_CRYPTO_CBC=y -+CONFIG_CRYPTO_CCM=y -+CONFIG_CRYPTO_CRC32C=y -+CONFIG_CRYPTO_CTR=y -+CONFIG_CRYPTO_DEFLATE=y -+CONFIG_CRYPTO_CMAC=y -+# CONFIG_CRYPTO_DEV_QCOM_ICE is not set -+CONFIG_CRYPTO_ECHAINIV=y -+CONFIG_CRYPTO_ECB=y -+CONFIG_CRYPTO_HASH2=y -+CONFIG_CRYPTO_HASH_INFO=y -+CONFIG_CRYPTO_HW=y -+CONFIG_CRYPTO_LZO=y -+CONFIG_CRYPTO_MANAGER=y -+CONFIG_CRYPTO_MANAGER2=y -+CONFIG_CRYPTO_NULL=y -+CONFIG_CRYPTO_RNG=y -+CONFIG_CRYPTO_RNG2=y -+CONFIG_CRYPTO_SEQIV=y -+CONFIG_CRYPTO_SHA256=y -+CONFIG_CRYPTO_WORKQUEUE=y -+CONFIG_CRYPTO_XZ=y -+CONFIG_CRYPTO_ARC4=y -+CONFIG_CRYPTO_GCM=y -+CONFIG_DCACHE_WORD_ACCESS=y -+CONFIG_DEBUG_BUGVERBOSE=y -+CONFIG_DEBUG_GPIO=y -+# CONFIG_DEBUG_INFO_REDUCED is not set -+CONFIG_DEBUG_LL_INCLUDE="mach/debug-macro.S" -+# CONFIG_DEBUG_MEM_USAGE is not set -+# CONFIG_DEBUG_UART_8250 is not set -+# CONFIG_DEBUG_USER is not set -+CONFIG_DECOMPRESS_GZIP=y -+CONFIG_DEVMEM=y -+# CONFIG_DIAG_OVER_USB is not set -+CONFIG_DMADEVICES=y -+CONFIG_DMA_ENGINE=y -+CONFIG_DMA_OF=y -+CONFIG_DMA_VIRTUAL_CHANNELS=y -+CONFIG_DTC=y -+# CONFIG_DWMAC_GENERIC is not set -+# CONFIG_DWMAC_IPQ806X is not set -+# CONFIG_DWMAC_SUNXI is not set -+# CONFIG_DW_DMAC_PCI is not set -+# CONFIG_VHOST_NET is not set -+# CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set -+CONFIG_DYNAMIC_DEBUG=y -+CONFIG_ETHERNET_PACKET_MANGLE=y -+CONFIG_EXT4_FS=y -+# CONFIG_EXT4_USE_FOR_EXT2 is not set -+CONFIG_FB=y -+CONFIG_FB_CFB_COPYAREA=y -+CONFIG_FB_CFB_FILLRECT=y -+CONFIG_FB_CFB_IMAGEBLIT=y -+CONFIG_FB_CMDLINE=y -+CONFIG_FB_QCOM_QPIC=y -+CONFIG_FB_QCOM_QPIC_ER_SSD1963_PANEL=y -+CONFIG_FB_SYS_FOPS=y -+CONFIG_FIXED_PHY=y -+CONFIG_FIX_EARLYCON_MEM=y -+CONFIG_FS_MBCACHE=y -+CONFIG_GENERIC_ALLOCATOR=y -+CONFIG_GENERIC_BUG=y -+CONFIG_GENERIC_CLOCKEVENTS=y -+CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y -+# CONFIG_GENERIC_CPUFREQ_KRAIT is not set -+CONFIG_GENERIC_IDLE_POLL_SETUP=y -+CONFIG_GENERIC_IO=y -+CONFIG_GENERIC_IRQ_SHOW=y -+CONFIG_GENERIC_IRQ_SHOW_LEVEL=y -+CONFIG_GENERIC_MSI_IRQ=y -+CONFIG_GENERIC_PCI_IOMAP=y -+CONFIG_GENERIC_PHY=y -+CONFIG_GENERIC_PINCONF=y -+CONFIG_GENERIC_SCHED_CLOCK=y -+CONFIG_GENERIC_SMP_IDLE_THREAD=y -+CONFIG_GENERIC_STRNCPY_FROM_USER=y -+CONFIG_GENERIC_STRNLEN_USER=y -+CONFIG_GENERIC_TIME_VSYSCALL=y -+CONFIG_GPIOLIB=y -+CONFIG_GPIOLIB_IRQCHIP=y -+CONFIG_GPIO_DEVRES=y -+# CONFIG_GPIO_LATCH is not set -+# CONFIG_GPIO_NXP_74HC153 is not set -+CONFIG_GPIO_SYSFS=y -+CONFIG_HANDLE_DOMAIN_IRQ=y -+CONFIG_HARDIRQS_SW_RESEND=y -+CONFIG_HAS_DMA=y -+CONFIG_HAS_IOMEM=y -+CONFIG_HAS_IOPORT_MAP=y -+# CONFIG_HAVE_64BIT_ALIGNED_ACCESS is not set -+CONFIG_HAVE_ARCH_AUDITSYSCALL=y -+CONFIG_HAVE_ARCH_BITREVERSE=y -+CONFIG_HAVE_ARCH_JUMP_LABEL=y -+CONFIG_HAVE_ARCH_KGDB=y -+CONFIG_HAVE_ARCH_PFN_VALID=y -+CONFIG_HAVE_ARCH_SECCOMP_FILTER=y -+CONFIG_HAVE_ARCH_TRACEHOOK=y -+CONFIG_HAVE_ARM_ARCH_TIMER=y -+# CONFIG_HAVE_BOOTMEM_INFO_NODE is not set -+CONFIG_HAVE_BPF_JIT=y -+CONFIG_HAVE_CC_STACKPROTECTOR=y -+CONFIG_HAVE_CLK=y -+CONFIG_HAVE_CLK_PREPARE=y -+CONFIG_HAVE_CONTEXT_TRACKING=y -+CONFIG_HAVE_C_RECORDMCOUNT=y -+CONFIG_HAVE_DEBUG_KMEMLEAK=y -+CONFIG_HAVE_DMA_API_DEBUG=y -+CONFIG_HAVE_DMA_ATTRS=y -+CONFIG_HAVE_DMA_CONTIGUOUS=y -+CONFIG_HAVE_DYNAMIC_FTRACE=y -+CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y -+CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y -+CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y -+CONFIG_HAVE_FUNCTION_TRACER=y -+# CONFIG_SRD_TRACE is not set -+CONFIG_HAVE_GENERIC_DMA_COHERENT=y -+CONFIG_HAVE_HW_BREAKPOINT=y -+CONFIG_HAVE_IDE=y -+CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y -+CONFIG_HAVE_KERNEL_GZIP=y -+CONFIG_HAVE_KERNEL_LZ4=y -+CONFIG_HAVE_KERNEL_LZMA=y -+CONFIG_HAVE_KERNEL_LZO=y -+CONFIG_HAVE_KERNEL_XZ=y -+# CONFIG_HAVE_KPROBES is not set -+# CONFIG_HAVE_KRETPROBES is not set -+CONFIG_HAVE_MEMBLOCK=y -+CONFIG_HAVE_MOD_ARCH_SPECIFIC=y -+CONFIG_HAVE_NET_DSA=y -+CONFIG_HAVE_OPROFILE=y -+# CONFIG_HAVE_OPTPROBES is not set -+CONFIG_HAVE_PERF_EVENTS=y -+CONFIG_HAVE_PERF_REGS=y -+CONFIG_HAVE_PERF_USER_STACK_DUMP=y -+CONFIG_HAVE_PROC_CPU=y -+CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y -+CONFIG_HAVE_SMP=y -+CONFIG_HAVE_SYSCALL_TRACEPOINTS=y -+CONFIG_HAVE_UID16=y -+CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y -+CONFIG_HIGHMEM=y -+CONFIG_HIGHPTE=y -+CONFIG_HOTPLUG_CPU=y -+CONFIG_HWMON=y -+CONFIG_HWSPINLOCK=y -+CONFIG_HWSPINLOCK_QCOM=y -+CONFIG_HW_RANDOM=y -+CONFIG_HW_RANDOM_MSM=y -+CONFIG_HZ_FIXED=0 -+CONFIG_I2C=y -+CONFIG_I2C_BOARDINFO=y -+CONFIG_I2C_CHARDEV=y -+CONFIG_I2C_COMPAT=y -+CONFIG_I2C_HELPER_AUTO=y -+CONFIG_I2C_QUP=y -+# CONFIG_IIO is not set -+# CONFIG_IIO_BUFFER is not set -+# CONFIG_IIO_TRIGGER is not set -+CONFIG_IKCONFIG=y -+CONFIG_IKCONFIG_PROC=y -+CONFIG_INITRAMFS_SOURCE="" -+# CONFIG_INPUT_PM8941_PWRKEY is not set -+CONFIG_IOMMU_HELPER=y -+# CONFIG_IOMMU_IO_PGTABLE_LPAE is not set -+# CONFIG_IPQ_DWC3_QTI_EXTCON is not set -+# CONFIG_IPQ_GCC_4019 is not set -+# CONFIG_IPQ_GCC_5018 is not set -+# CONFIG_IPQ_APSS_5018 is not set -+# CONFIG_IPQ_GCC_6018 is not set -+# CONFIG_IPQ_APSS_6018 is not set -+# CONFIG_IPQ_GCC_806X is not set -+# CONFIG_IPQ_ADSS_807x is not set -+# CONFIG_IPQ_APSS_807x is not set -+# CONFIG_IPQ_GCC_807x is not set -+# CONFIG_IPQ_ADCC_4019 is not set -+# CONFIG_IPQ_LCC_806X is not set -+# CONFIG_IPQ_REMOTEPROC_ADSP is not set -+# CONFIG_IPQ_SUBSYSTEM_RESTART is not set -+# CONFIG_IPQ_SUBSYSTEM_RESTART_TEST is not set -+CONFIG_IRQCHIP=y -+CONFIG_IRQ_DOMAIN=y -+CONFIG_IRQ_DOMAIN_HIERARCHY=y -+CONFIG_IRQ_FORCED_THREADING=y -+CONFIG_IRQ_WORK=y -+CONFIG_JBD2=y -+# CONFIG_IPC_ROUTER is not set -+# CONFIG_IPC_ROUTER_SECURITY is not set -+# CONFIG_IPC_LOGGING is not set -+CONFIG_KPSS_XCC=y -+# CONFIG_KRAITCC is not set -+# CONFIG_KRAIT_CLOCKS is not set -+# CONFIG_KRAIT_L2_ACCESSORS is not set -+CONFIG_LEDS_IPQ=y -+CONFIG_LEDS_TLC591XX=y -+# CONFIG_LEDS_PCA9956B is not set -+CONFIG_LIBFDT=y -+CONFIG_LOCKUP_DETECTOR=y -+CONFIG_LOCK_SPIN_ON_OWNER=y -+CONFIG_LZO_COMPRESS=y -+CONFIG_LZO_DECOMPRESS=y -+CONFIG_MDIO=y -+CONFIG_MDIO_BITBANG=y -+CONFIG_MDIO_BOARDINFO=y -+CONFIG_MDIO_GPIO=y -+# CONFIG_MDIO_QCA is not set -+CONFIG_MFD_QCOM_RPM=y -+CONFIG_MFD_SPMI_PMIC=y -+# CONFIG_SLIMBUS is not set -+# CONFIG_SLIMBUS_MSM_CTRL is not set -+# CONFIG_SLIMBUS_MSM_NGD is not set -+# CONFIG_OF_SLIMBUS is not set -+CONFIG_MFD_SYSCON=y -+CONFIG_MIGHT_HAVE_CACHE_L2X0=y -+CONFIG_MIGHT_HAVE_PCI=y -+CONFIG_MMC=y -+CONFIG_MMC_ARMMMCI=y -+CONFIG_MMC_BLOCK=y -+CONFIG_MMC_BLOCK_MINORS=32 -+CONFIG_MMC_QCOM_DML=y -+CONFIG_MMC_QCOM_TUNING=y -+CONFIG_MMC_SDHCI=y -+CONFIG_MMC_SDHCI_MSM=y -+# CONFIG_MMC_SDHCI_OF_ARASAN is not set -+# CONFIG_MMC_SDHCI_PCI is not set -+CONFIG_MMC_SDHCI_PLTFM=y -+# CONFIG_MMC_TIFM_SD is not set -+CONFIG_MODULES_TREE_LOOKUP=y -+CONFIG_MODULES_USE_ELF_REL=y -+# CONFIG_MPLS_ROUTING is not set -+# CONFIG_MSM_GCC_8660 is not set -+# CONFIG_MSM_GCC_8916 is not set -+# CONFIG_MSM_GCC_8960 is not set -+# CONFIG_MSM_GCC_8974 is not set -+# CONFIG_MSM_LCC_8960 is not set -+# CONFIG_MSM_MMCC_8960 is not set -+# CONFIG_MSM_MMCC_8974 is not set -+# CONFIG_MSM_MHI is not set -+# CONFIG_MSM_IPC_ROUTER_MHI_XPRT is not set -+# CONFIG_MSM_MHI_DEBUG is not set -+# CONFIG_MSM_MHI_DEV is not set -+# CONFIG_MSM_MHI_UCI is not set -+# CONFIG_DIAGFWD_BRIDGE_CODE is not set -+# CONFIG_MSM_BUS_SCALING is not set -+# CONFIG_BUS_TOPOLOGY_ADHOC is not set -+# CONFIG_QPNP_REVID is not set -+# CONFIG_SPS is not set -+# CONFIG_SPS_SUPPORT_NDP_BAM is not set -+# CONFIG_USB_BAM is not set -+# CONFIG_SPS_SUPPORT_BAMDMA is not set -+# CONFIG_IPA is not set -+# CONFIG_IPA3 is not set -+# CONFIG_EP_PCIE is not set -+# CONFIG_GSI is not set -+# CONFIG_PFT is not set -+# CONFIG_SEEMP_CORE is not set -+# CONFIG_GPIO_USB_DETECT is not set -+# CONFIG_MSM_GLINK is not set -+# CONFIG_MSM_GLINK_LOOPBACK_SERVER is not set -+# CONFIG_MSM_GLINK_SMEM_NATIVE_XPRT is not set -+# CONFIG_MSM_GLINK_PKT is not set -+# CONFIG_MSM_IPC_ROUTER_GLINK_XPRT is not set -+# CONFIG_MSM_QMI_INTERFACE is not set -+# CONFIG_MSM_TEST_QMI_CLIENT is not set -+# CONFIG_GLINK_DEBUG_FS is not set -+# CONFIG_MSM_RPM_SMD is not set -+# CONFIG_MSM_RPM_GLINK is not set -+CONFIG_MSM_RPM_LOG=y -+# CONFIG_MSM_SMEM is not set -+CONFIG_MTD_CMDLINE_PARTS=y -+CONFIG_MTD_M25P80=y -+CONFIG_MTD_NAND=y -+CONFIG_MTD_NAND_ECC=y -+CONFIG_MTD_NAND_QCOM=y -+CONFIG_MTD_QCOM_SMEM_PARTS=y -+CONFIG_MTD_SPINAND_GIGADEVICE=y -+CONFIG_MTD_SPINAND_MT29F=y -+CONFIG_MTD_SPINAND_ONDIEECC=y -+CONFIG_MTD_SPI_NOR=y -+CONFIG_MTD_SPLIT_FIRMWARE=y -+CONFIG_MTD_SPLIT_FIT_FW=y -+CONFIG_MTD_UBI=y -+CONFIG_MTD_UBI_BEB_LIMIT=20 -+CONFIG_MTD_UBI_BLOCK=y -+# CONFIG_MTD_UBI_FASTMAP is not set -+CONFIG_MTD_UBI_GLUEBI=y -+CONFIG_MTD_UBI_WL_THRESHOLD=4096 -+CONFIG_MULTI_IRQ_HANDLER=y -+CONFIG_MUTEX_SPIN_ON_OWNER=y -+CONFIG_NEED_DMA_MAP_STATE=y -+CONFIG_NEON=y -+CONFIG_NET=y -+# CONFIG_NET_DSA_MV88E6063 is not set -+CONFIG_NET_FLOW_LIMIT=y -+CONFIG_NET_PTP_CLASSIFY=y -+CONFIG_NO_BOOTMEM=y -+CONFIG_NO_HZ=y -+CONFIG_NO_HZ_COMMON=y -+CONFIG_NO_HZ_IDLE=y -+CONFIG_NR_CPUS=4 -+CONFIG_NUM_ALT_PARTITION=8 -+CONFIG_NVMEM=y -+CONFIG_OF=y -+CONFIG_OF_ADDRESS=y -+CONFIG_OF_ADDRESS_PCI=y -+CONFIG_OF_EARLY_FLATTREE=y -+CONFIG_OF_FLATTREE=y -+CONFIG_OF_GPIO=y -+CONFIG_OF_IRQ=y -+CONFIG_OF_MDIO=y -+CONFIG_OF_MTD=y -+CONFIG_OF_NET=y -+CONFIG_OF_PCI=y -+CONFIG_OF_PCI_IRQ=y -+CONFIG_OF_RESERVED_MEM=y -+CONFIG_OLD_SIGACTION=y -+CONFIG_OLD_SIGSUSPEND3=y -+CONFIG_PAGE_OFFSET=0xC0000000 -+CONFIG_PANIC_ON_OOPS=y -+CONFIG_PANIC_ON_OOPS_VALUE=1 -+CONFIG_PANIC_TIMEOUT=5 -+CONFIG_PCI=y -+# CONFIG_PCIEAER is not set -+CONFIG_PCIE_DW=y -+# CONFIG_PCIE_DW_PLAT is not set -+CONFIG_PCIE_PME=y -+CONFIG_PCIE_QCOM=y -+CONFIG_PCI_DOMAINS=y -+CONFIG_PCI_DOMAINS_GENERIC=y -+CONFIG_PCI_MSI=y -+CONFIG_PERF_EVENTS=y -+CONFIG_PERF_USE_VMALLOC=y -+CONFIG_PGTABLE_LEVELS=2 -+CONFIG_PHYLIB=y -+# CONFIG_PHY_IPQ_BALDUR_USB is not set -+# CONFIG_PHY_IPQ_UNIPHY_USB is not set -+# CONFIG_PHY_QCOM_APQ8064_SATA is not set -+# CONFIG_PHY_QCOM_IPQ806X_SATA is not set -+CONFIG_PHY_QCA_PCIE_QMP=y -+# CONFIG_PHY_QCOM_UFS is not set -+# CONFIG_PHY_IPQ_UNIPHY_PCIE is not set -+CONFIG_PINCTRL=y -+# CONFIG_PINCTRL_APQ8064 is not set -+# CONFIG_PINCTRL_APQ8084 is not set -+# CONFIG_PINCTRL_IPQ4019 is not set -+# CONFIG_PINCTRL_IPQ6018 is not set -+# CONFIG_PINCTRL_IPQ8064 is not set -+# CONFIG_PINCTRL_IPQ807x is not set -+# CONFIG_PINCTRL_IPQ5018 is not set -+CONFIG_PINCTRL_MSM=y -+# CONFIG_PINCTRL_MSM8660 is not set -+# CONFIG_PINCTRL_MSM8916 is not set -+# CONFIG_PINCTRL_MSM8960 is not set -+CONFIG_PINCTRL_QCOM_SPMI_PMIC=y -+# CONFIG_PINCTRL_QCOM_SSBI_PMIC is not set -+# CONFIG_PL330_DMA is not set -+CONFIG_PM=y -+CONFIG_PM_CLK=y -+# CONFIG_PM_DEBUG is not set -+CONFIG_PM_GENERIC_DOMAINS=y -+CONFIG_PM_GENERIC_DOMAINS_OF=y -+CONFIG_PM_GENERIC_DOMAINS_SLEEP=y -+CONFIG_PM_OPP=y -+CONFIG_PM_SLEEP=y -+CONFIG_PM_SLEEP_SMP=y -+CONFIG_POWER_RESET=y -+CONFIG_POWER_RESET_MSM=y -+CONFIG_POWER_SUPPLY=y -+CONFIG_PREEMPT=y -+CONFIG_PREEMPT_COUNT=y -+# CONFIG_PREEMPT_NONE is not set -+CONFIG_PREEMPT_RCU=y -+CONFIG_PRINTK_TIME=y -+CONFIG_PROC_PAGE_MONITOR=y -+# CONFIG_PROC_STRIPPED is not set -+CONFIG_PSTORE=y -+CONFIG_PSTORE_RAM=y -+# CONFIG_PSTORE_CONSOLE is not set -+# CONFIG_PSTORE_PMSG is not set -+CONFIG_PWM=y -+# CONFIG_PWM_IPQ4019 is not set -+# CONFIG_PWM_PCA9685 is not set -+CONFIG_PWM_SYSFS=y -+CONFIG_QCOM_ADM=y -+# CONFIG_QCOM_APM is not set -+CONFIG_QCOM_BAM_DMA=y -+# CONFIG_QTI_BT_TTY is not set -+# CONFIG_QCOM_COINCELL is not set -+# CONFIG_QCOM_DCC is not set -+CONFIG_QCOM_GDSC=y -+CONFIG_QCOM_GSBI=y -+# CONFIG_QCOM_HFPLL is not set -+# CONFIG_QCOM_MEMORY_DUMP_V2 is not set -+# CONFIG_QCOM_MDT_LOADER is not set -+CONFIG_QCOM_QFPROM=y -+# CONFIG_QCOM_SPMI_TEMP_ALARM is not set -+CONFIG_QCOM_RPM_CLK=y -+# CONFIG_QCOM_RTB is not set -+# CONFIG_QCOM_PM is not set -+CONFIG_QCOM_SCM=y -+CONFIG_QCOM_SCM_32=y -+# CONFIG_HAVE_ARM_SMCCC is not set -+CONFIG_QCA_SCM_RESTART_REASON=y -+CONFIG_IPQ_TCSR=y -+CONFIG_QCOM_QFPROM=y -+# CONFIG_QCOM_SMD is not set -+CONFIG_QCOM_SMEM=y -+CONFIG_QCOM_SMEM_STATE=y -+# CONFIG_QCOM_SMD is not set -+CONFIG_QCOM_SMP2P=y -+# CONFIG_QCOM_SPMI_VADC is not set -+CONFIG_QCOM_TSENS=y -+CONFIG_QCOM_TZ_LOG=y -+CONFIG_QCOM_WDT=y -+CONFIG_QMI_ENCDEC=y -+CONFIG_RATIONAL=y -+# CONFIG_RCU_BOOST is not set -+CONFIG_RCU_CPU_STALL_TIMEOUT=21 -+# CONFIG_RCU_EXPERT is not set -+CONFIG_RCU_STALL_COMMON=y -+CONFIG_RD_GZIP=y -+CONFIG_REGMAP=y -+# CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS is not set -+CONFIG_REGMAP_MMIO=y -+CONFIG_REGULATOR=y -+# CONFIG_REGULATOR_CPR3 is not set -+CONFIG_REGULATOR_GPIO=y -+CONFIG_REGULATOR_FIXED_VOLTAGE=y -+CONFIG_REGULATOR_QCOM_RPM=y -+CONFIG_REGULATOR_QCOM_SPMI=y -+# CONFIG_REGULATOR_IPQ40XX is not set -+# CONFIG_REGULATOR_RPM_SMD is not set -+# CONFIG_REGULATOR_RPM_GLINK is not set -+CONFIG_RELAY=y -+CONFIG_REMOTEPROC=y -+# CONFIG_IPQ807X_REMOTEPROC is not set -+CONFIG_RESET_CONTROLLER=y -+CONFIG_RFS_ACCEL=y -+CONFIG_RPS=y -+CONFIG_RTC_CLASS=y -+CONFIG_NET_L3_MASTER_DEV=y -+# CONFIG_RTC_DRV_CMOS is not set -+# CONFIG_RTC_DRV_PM8XXX is not set -+CONFIG_RWSEM_SPIN_ON_OWNER=y -+CONFIG_RWSEM_XCHGADD_ALGORITHM=y -+# CONFIG_SATA_AHCI is not set -+CONFIG_SCHED_HRTICK=y -+# CONFIG_SCSI is not set -+# CONFIG_SCHED_INFO is not set -+# CONFIG_SCSI_DMA is not set -+# CONFIG_SERIAL_8250 is not set -+# CONFIG_SERIAL_8250_CONSOLE is not set -+# CONFIG_SERIAL_8250_DMA is not set -+# CONFIG_SERIAL_AMBA_PL010 is not set -+# CONFIG_SERIAL_AMBA_PL011 is not set -+CONFIG_SERIAL_MSM=y -+CONFIG_SERIAL_MSM_CONSOLE=y -+# CONFIG_VIRTIO_CONSOLE is not set -+CONFIG_SMP=y -+CONFIG_SMP_ON_UP=y -+# CONFIG_SND is not set -+CONFIG_SND_DYNAMIC_MINORS=y -+CONFIG_SND_MAX_CARDS=32 -+CONFIG_SND_PROC_FS=y -+# CONFIG_SND_COMPRESS_OFFLOAD is not set -+CONFIG_SND_PCM=y -+CONFIG_SND_SOC=y -+# CONFIG_SND_SOC_APQ8016_SBC is not set -+CONFIG_SND_SOC_I2C_AND_SPI=y -+CONFIG_SND_SOC_IPQ=y -+# CONFIG_SND_SOC_IPQ806X_LPAIF is not set -+# CONFIG_SND_SOC_IPQ806X_PCM_RAW is not set -+CONFIG_SND_SOC_IPQ_ADSS=y -+CONFIG_SND_SOC_IPQ_CODEC=y -+CONFIG_SND_SOC_IPQ_CPU_DAI=y -+CONFIG_SND_SOC_IPQ_MBOX=y -+CONFIG_SND_SOC_IPQ_PCM_I2S=y -+CONFIG_SND_SOC_IPQ_PCM_RAW=y -+CONFIG_SND_SOC_IPQ_PCM_SPDIF=y -+CONFIG_SND_SOC_IPQ_PCM_TDM=y -+CONFIG_SND_SOC_IPQ_STEREO=y -+CONFIG_SND_SOC_QCOM=y -+# CONFIG_SND_SOC_STORM is not set -+CONFIG_SOUND=y -+CONFIG_SPARSE_IRQ=y -+CONFIG_SPI=y -+CONFIG_SPI_MASTER=y -+CONFIG_SPI_QUP=y -+CONFIG_SPI_SPIDEV=y -+# CONFIG_SPI_VSC7385 is not set -+CONFIG_SPMI=y -+CONFIG_SPMI_MSM_PMIC_ARB=y -+CONFIG_SRCU=y -+# CONFIG_STRIP_ASM_SYMS is not set -+# CONFIG_STOPWATCH is not set -+CONFIG_SUSPEND=y -+CONFIG_SWCONFIG=y -+CONFIG_SWIOTLB=y -+# CONFIG_SWAP is not set -+CONFIG_SWP_EMULATE=y -+CONFIG_SYS_SUPPORTS_APM_EMULATION=y -+CONFIG_THERMAL=y -+CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE=y -+CONFIG_THERMAL_GOV_STEP_WISE=y -+CONFIG_THERMAL_GOV_USER_SPACE=y -+CONFIG_THERMAL_HWMON=y -+CONFIG_THERMAL_OF=y -+CONFIG_THERMAL_WRITABLE_TRIPS=y -+# CONFIG_THUMB2_KERNEL is not set -+# CONFIG_TICK_CPU_ACCOUNTING is not set -+CONFIG_IRQ_TIME_ACCOUNTING=y -+CONFIG_TRACING_EVENTS_GPIO=y -+CONFIG_UBIFS_FS=y -+CONFIG_UBIFS_FS_ADVANCED_COMPR=y -+CONFIG_UBIFS_FS_LZO=y -+CONFIG_UBIFS_FS_XZ=y -+CONFIG_UBIFS_FS_ZLIB=y -+CONFIG_UEVENT_HELPER_PATH="" -+CONFIG_UNCOMPRESS_INCLUDE="debug/uncompress.h" -+CONFIG_UNINLINE_SPIN_UNLOCK=y -+CONFIG_USB_GADGET=n -+CONFIG_USB_SUPPORT=y -+# CONFIG_USB_DWC3_OF_SIMPLE is not set -+# CONFIG_USB_QCOM_8X16_PHY is not set -+# CONFIG_USB_QCOM_KS_BRIDGE is not set -+# CONFIG_USB_QCOM_QUSB_PHY is not set -+# CONFIG_USB_QCOM_QMP_PHY is not set -+# CONFIG_USB_QCA_M31_PHY is not set -+# CONFIG_USB_EHCI_ROOT_HUB_TT is not set -+# CONFIG_USB_EHCI_TT_NEWSCHED is not set -+# CONFIG_USB_OHCI_LITTLE_ENDIAN is not set -+CONFIG_USE_OF=y -+CONFIG_VDSO=y -+CONFIG_VECTORS_BASE=0xffff0000 -+CONFIG_VFP=y -+CONFIG_VFPv3=y -+CONFIG_VM_EVENT_COUNTERS=y -+CONFIG_WATCHDOG_CORE=y -+# CONFIG_WL_TI is not set -+# CONFIG_WQ_POWER_EFFICIENT_DEFAULT is not set -+CONFIG_XPS=y -+CONFIG_XZ_DEC_ARM=y -+CONFIG_XZ_DEC_BCJ=y -+CONFIG_ZBOOT_ROM_BSS=0 -+CONFIG_ZBOOT_ROM_TEXT=0 -+CONFIG_ZLIB_DEFLATE=y -+CONFIG_ZLIB_INFLATE=y -+CONFIG_ZONE_DMA_FLAG=0 -+CONFIG_QCOM_CACHE_DUMP=y -+CONFIG_QCOM_CACHE_DUMP_ON_PANIC=y -+# CONFIG_QCOM_RESTART_REASON is not set -+# CONFIG_QCOM_DLOAD_MODE is not set -+CONFIG_FW_AUTH=y -+CONFIG_FW_AUTH_TEST=m -+CONFIG_ASYMMETRIC_KEY_TYPE=y -+CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y -+CONFIG_PUBLIC_KEY_ALGO_RSA=y -+CONFIG_X509_CERTIFICATE_PARSER=y -+CONFIG_PKCS7_MESSAGE_PARSER=n -+CONFIG_KEYS=y -+# CONFIG_SKB_RECYCLER is not set -+CONFIG_SKB_RECYCLER_MULTI_CPU=y -+# CONFIG_SKB_RECYCLER_PREALLOC is not set -+# CONFIG_U_SERIAL_CONSOLE is not set -+CONFIG_SCSI_SCAN_ASYNC=y -+# CONFIG_NF_IPV6_DUMMY_HEADER is not set -+# CONFIG_RMNET is not set -+# CONFIG_RMNET_DATA is not set -+# CONFIG_RMNET_CTL is not set -+# CONFIG_MSM_SECURE_BUFFER is not set -+# CONFIG_STAGING is not set -+# CONFIG_ANDROID is not set -+# CONFIG_ION is not set -+# CONFIG_ION_DUMMY is not set -+# CONFIG_ION_MSM is not set -+# CONFIG_ION_TEST is not set -+# CONFIG_CMA is not set -+# CONFIG_CMA_DEBUG is not set -+# CONFIG_CMA_DEBUGFS is not set -+# CONFIG_DMA_CMA is not set -+# CONFIG_CMA_AREAS is not set -+# CONFIG_CMA_SIZE_MBYTES is not set -+# CONFIG_CMA_SIZE_SEL_MBYTES is not set -+# CONFIG_CMA_SIZE_SEL_PERCENTAGE is not set -+# CONFIG_CMA_SIZE_SEL_MIN is not set -+# CONFIG_CMA_SIZE_SEL_MAX is not set -+# CONFIG_CMA_ALIGNMENT is not set -+# CONFIG_ASHMEM is not set -+# CONFIG_ANDROID_TIMED_OUTPUT is not set -+# CONFIG_ANDROID_LOW_MEMORY_KILLER is not set -+# CONFIG_SYNC is not set -+# CONFIG_SW_SYNC is not set -+# CONFIG_FSL_MC_BUS is not set -+# CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS is not set -+CONFIG_ALLOC_SKB_PAGE_FRAG_DISABLE=y -+# CONFIG_MMAP_ALLOW_UNINITIALIZED is not set -+# CONFIG_MAILBOX is not set -+# CONFIG_MAILBOX_TEST is not set -+# CONFIG_QCOM_APCS_IPC is not set -+# CONFIG_QCOM_GLINK_SSR is not set -+# CONFIG_QCOM_Q6V5_WCSS is not set -+# CONFIG_QCOM_SYSMON is not set -+# CONFIG_QRTR is not set -+# CONFIG_QRTR_SMD is not set -+# CONFIG_QRTR_TUN is not set -+# CONFIG_RPMSG is not set -+# CONFIG_RPMSG_QCOM_GLINK_RPM is not set -+# CONFIG_RPMSG_VIRTIO is not set -+# CONFIG_RPMSG_CHAR is not set -+# CONFIG_RPMSG_QCOM_GLINK_SMEM is not set -+# CONFIG_RPMSG_QCOM_SMD is not set -+CONFIG_QCA_MINIDUMP=y -+# CONFIG_QCA_MINIDUMP_DEBUG is not set -+# CONFIG_QRTR_USB is not set -+# CONFIG_QRTR_FIFO is not set -+CONFIG_QRTR_MHI=y -+CONFIG_MHI_BUS=y -+# CONFIG_MHI_QTI is not set -+# CONFIG_MHI_NETDEV is not set -+# CONFIG_MHI_DEBUG is not set -+# CONFIG_MHI_UCI is not set -+# CONFIG_MHI_SATELLITE is not set -+# CONFIG_DIAG_OVER_QRTR is not set -+# CONFIG_MSM_ADSPRPC is not set -+CONFIG_CRYPTO_MICHAEL_MIC=y -+# CONFIG_ARCH_HAS_KCOV is not set -+# CONFIG_KCOV is not set -+# CONFIG_GCC_PLUGINS is not set -+# CONFIG_QTI_Q6V5_ADSP is not set -+# CONFIG_MSM_RPM_RPMSG is not set -+# CONFIG_RPMSG_QCOM_GLINK_RPM is not set -+# CONFIG_REGULATOR_RPM_GLINK is not set -+# CONFIG_MTD_NAND_SERIAL is not set -+# CONFIG_ARM_QTI_IPQ60XX_CPUFREQ is not set -+# CONFIG_PAGE_SCOPE_MULTI_PAGE_READ is not set -+# CONFIG_CRYPTO_NO_ZERO_LEN_HASH is not set -+# CONFIG_CRYPTO_DISABLE_AES192_TEST is not set -+# CONFIG_QTI_EUD is not set -+# CONFIG_EUD_EXTCON_SUPPORT is not set -+# CONFIG_CLK_TEST_5018 is not set -+CONFIG_MAP_E_SUPPORT=y -+# CONFIG_IPQ_FLASH_16M_PROFILE is not set -+# CONFIG_QGIC2_MSI is not set -+CONFIG_BRIDGE_VLAN_FILTERING=y -+CONFIG_LEDS_GPIO=y -+# CONFIG_ARCH_IPQ256M is not set -+CONFIG_SKB_FIXED_SIZE_2K=y -+# CONFIG_IPQ_MEM_PROFILE is not set -+# CONFIG_VIRTIO_NET is not set -+# CONFIG_QCA_85XX_SWITCH is not set -+CONFIG_AQ_PHY=y -+CONFIG_DIAG_CHAR=y -+# CONFIG_HW_RANDOM_VIRTIO is not set -+# CONFIG_BOOTCONFIG_PARTITION is not set -+# CONFIG_CRYPTO_DEV_QCEDEV is not set -+# CONFIG_CRYPTO_DEV_QCRYPTO is not set -+# CONFIG_MHI_BUS_TEST is not set -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts -new file mode 100644 -index 0000000000..5f2d4fb03f ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts -new file mode 100644 -index 0000000000..ffd7899228 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-cig-wf188n.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts -new file mode 100644 -index 0000000000..531510a764 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-edgecore-eap101.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-miwifi-ax1800.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-miwifi-ax1800.dts -new file mode 100644 -index 0000000000..5c452c5741 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-miwifi-ax1800.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-miwifi-ax1800.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts -new file mode 100644 -index 0000000000..1250eb19f8 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2020, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts -new file mode 100644 -index 0000000000..d1d3f3755e ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap102.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts -new file mode 100644 -index 0000000000..1527f81a40 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-eap106.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts -new file mode 100644 -index 0000000000..f3b25e2633 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex227.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts -new file mode 100644 -index 0000000000..7cfd66ba0a ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-ex447.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-sercomm-wallaby.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-sercomm-wallaby.dts -new file mode 100644 -index 0000000000..e756aa5622 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-sercomm-wallaby.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-sercomm-wallaby.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts -new file mode 100644 -index 0000000000..4274b6c704 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts -new file mode 100644 -index 0000000000..372f29df38 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq807x-wf194c4.dts -@@ -0,0 +1,26 @@ -+/* -+ * Copyright (c) 2017, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts" -+ -+/ { -+ soc { -+ pmu { -+ compatible = "arm,cortex-a7-pmu"; -+ interrupts = ; -+ }; -+ }; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-miwifi-ax1800.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-miwifi-ax1800.dts -new file mode 100755 -index 0000000000..ba4a5ef3ca ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-miwifi-ax1800.dts -@@ -0,0 +1,419 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#define __IPQ_MEM_PROFILE_256_MB__ -+#include "qcom-ipq6018.dtsi" -+#include "qcom-ipq6018-rpm-regulator.dtsi" -+#include "qcom-ipq6018-cpr-regulator.dtsi" -+#include "qcom-ipq6018-cp-cpu.dtsi" -+#include -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ model = "Qualcomm Technologies, Inc. IPQ6018/AP-CP03-C1"; -+ compatible = "xiaomi,ax1800", "qcom,ipq6018-cp03", "qcom,ipq6018"; -+ interrupt-parent = <&intc>; -+ qcom,msm-id = <0x1A5 0x0>; -+ -+ aliases { -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet1 = "/soc/dp2"; -+ ethernet2 = "/soc/dp3"; -+ ethernet3 = "/soc/dp4"; -+ ethernet4 = "/soc/dp5"; -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -+ bootargs-append = " swiotlb=1 coherent_pool=2M console=ttyMSM0,115200,n8"; -+ }; -+ -+ /* -+ * +=========+==============+========================+ -+ * | | | | -+ * | Region | Start Offset | Size | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | | | | -+ * | | | | -+ * | | | | -+ * | | | | -+ * | Linux | 0x41000000 | 139MB | -+ * | | | | -+ * | | | | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | TZ App | 0x49B00000 | 6MB | -+ * +--------+--------------+-------------------------+ -+ * -+ * From the available 145 MB for Linux in the first 256 MB, -+ * we are reserving 6 MB for TZAPP. -+ * -+ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+ * for memory layout. -+ */ -+ -+}; -+ -+&tlmm { -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio44", "gpio45"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ wps_button { -+ pins = "gpio19"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio64"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_2 { -+ pins = "gpio75"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ }; -+ -+ gpio_keys { -+ pinctrl-names = "default"; -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ -+ button@1 { -+ debounce-interval = <0x3c>; -+ label = "reset"; -+ linux,code = ; -+ linux,input-type = <0x01>; -+ gpios = <&tlmm 0x13 GPIO_ACTIVE_LOW>; -+ }; -+ }; -+ leds_pins: leds_pins { -+ led_blue { -+ pins = "gpio31"; -+ bias-pull-down; -+ function = "gpio"; -+ drive-strength = <0x08>; -+ }; -+ -+ net_blue { -+ pins = "gpio29"; -+ bias-pull-down; -+ function = "gpio"; -+ drive-strength = <0x08>; -+ }; -+ -+ net_yellow { -+ pins = "gpio30"; -+ bias-pull-down; -+ function = "gpio"; -+ drive-strength = <0x08>; -+ }; -+ -+ led_yellow { -+ pins = "gpio32"; -+ bias-pull-down; -+ function = "gpio"; -+ drive-strength = <0x08>; -+ }; -+ }; -+}; -+ -+&soc { -+ mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 75 0>; -+ status = "ok"; -+ ethernet-phy@0 { -+ reg = <0x00>; -+ }; -+ -+ ethernet-phy@1 { -+ reg = <0x01>; -+ }; -+ -+ ethernet-phy@4 { -+ reg = <0x04>; -+ }; -+ -+ ethernet-phy@2 { -+ reg = <0x02>; -+ }; -+ -+ ethernet-phy@3 { -+ reg = <0x03>; -+ }; -+ }; -+ -+ ess-switch@3a000000 { -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x1e>; /* lan port bitmap */ -+ switch_wan_bmp = <0x20>; /* wan port bitmap */ -+ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -+ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -+ qcom,port_phyinfo { -+ port@0 { -+ port_id = <0x01>; -+ phy_address = <0x00>; -+ }; -+ -+ port@4 { -+ port_id = <0x05>; -+ phy_address = <0x04>; -+ }; -+ -+ port@2 { -+ port_id = <0x03>; -+ phy_address = <0x02>; -+ }; -+ -+ port@1 { -+ port_id = <0x02>; -+ phy_address = <0x01>; -+ }; -+ -+ port@3 { -+ port_id = <0x04>; -+ phy_address = <0x03>; -+ }; -+ -+ }; -+ }; -+ -+ dp2 { -+ reg = <0x3a001200 0x200>; -+ qcom,id = <0x02>; -+ qcom,phy-mdio-addr = <0x01>; -+ qcom,link-poll = <0x01>; -+ qcom,mactype = <0x00>; -+ compatible = "qcom,nss-dp"; -+ phy-mode = "sgmii"; -+ local-mac-address = [000000000000]; -+ device_type = "network"; -+ }; -+ -+ dp3 { -+ reg = <0x3a001400 0x200>; -+ qcom,id = <0x03>; -+ qcom,phy-mdio-addr = <0x02>; -+ qcom,link-poll = <0x01>; -+ qcom,mactype = <0x00>; -+ compatible = "qcom,nss-dp"; -+ phy-mode = "sgmii"; -+ local-mac-address = [000000000000]; -+ device_type = "network"; -+ }; -+ -+ dp4 { -+ reg = <0x3a001600 0x200>; -+ qcom,id = <0x04>; -+ qcom,phy-mdio-addr = <0x03>; -+ qcom,link-poll = <0x01>; -+ qcom,mactype = <0x00>; -+ compatible = "qcom,nss-dp"; -+ phy-mode = "sgmii"; -+ local-mac-address = [000000000000]; -+ device_type = "network"; -+ }; -+ -+ dp5 { -+ reg = <0x3a001800 0x200>; -+ qcom,id = <0x05>; -+ qcom,phy-mdio-addr = <0x04>; -+ qcom,link-poll = <0x01>; -+ qcom,mactype = <0x00>; -+ compatible = "qcom,nss-dp"; -+ phy-mode = "sgmii"; -+ local-mac-address = [8c 53 c3 b6 7d ac]; -+ device_type = "network"; -+ }; -+ -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ -+ led_sys_yellow { -+ label = "yellow:status"; -+ default-state = "on"; -+ gpio = <&tlmm 0x1e GPIO_ACTIVE_HIGH>; -+ }; -+ -+ led_net_yellow { -+ label = "yellow:net"; -+ default-state = "off"; -+ gpio = <&tlmm 0x20 GPIO_ACTIVE_HIGH>; -+ }; -+ -+ led_sys_blue { -+ label = "blue:status"; -+ default-state = "off"; -+ gpio = <&tlmm 0x1d GPIO_ACTIVE_HIGH>; -+ }; -+ -+ led_net_blue { -+ label = "blue:net"; -+ default-state = "off"; -+ gpio = <&tlmm 0x1f GPIO_ACTIVE_HIGH>; -+ }; -+ }; -+}; -+ -+&blsp1_uart3 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+ -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&usb3 { -+ status = "ok"; -+}; -+ -+&nss_crypto { -+ status = "ok"; -+}; -+ -+&q6_region { -+ reg = <0x0 0x4ab00000 0x0 0x05500000>; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts -new file mode 100755 -index 0000000000..12065b62f6 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018.dts -@@ -0,0 +1,441 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "qcom-ipq6018.dtsi" -+#include "qcom-ipq6018-rpm-regulator.dtsi" -+#include "qcom-ipq6018-cpr-regulator.dtsi" -+#include "qcom-ipq6018-cp-cpu.dtsi" -+#include -+#include -+ -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ model = "Wallys DR6018"; -+ compatible = "wallys,dr6018", "qcom,ipq6018-cp01", "qcom,ipq6018"; -+ interrupt-parent = <&intc>; -+ -+ aliases { -+ serial0 = &blsp1_uart3; -+ serial1 = &blsp1_uart2; -+ -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet0 = "/soc/dp1"; -+ ethernet1 = "/soc/dp2"; -+ -+ led-boot = &led_power; -+ led-failsafe = &led_power; -+ led-running = &led_power; -+ led-upgrade = &led_power; -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -+ bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; -+ }; -+ -+ /* -+ * +=========+==============+========================+ -+ * | | | | -+ * | Region | Start Offset | Size | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | | | | -+ * | | | | -+ * | | | | -+ * | | | | -+ * | Linux | 0x41000000 | 139MB | -+ * | | | | -+ * | | | | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | TZ App | 0x49B00000 | 6MB | -+ * +--------+--------------+-------------------------+ -+ * -+ * From the available 145 MB for Linux in the first 256 MB, -+ * we are reserving 6 MB for TZAPP. -+ * -+ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+ * for memory layout. -+ */ -+ -+/* TZAPP is enabled only in default memory profile */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+ reserved-memory { -+ tzapp:tzapp@49B00000 { /* TZAPPS */ -+ no-map; -+ reg = <0x0 0x49B00000 0x0 0x00600000>; -+ }; -+ }; -+#endif -+}; -+ -+&tlmm { -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio44", "gpio45"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ extcon_usb_pins: extcon_usb_pins { -+ mux { -+ pins = "gpio26"; -+ function = "gpio"; -+ drive-strength = <2>; -+ bias-pull-down; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ wps_button { -+ pins = "gpio19"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio64"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_2 { -+ pins = "gpio75"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ }; -+ -+ leds_pins: leds_pins { -+ led_pwr { -+ pins = "gpio74"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_5g { -+ pins = "gpio35"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_2g { -+ pins = "gpio37"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ uart2_pins: uart2_pins { -+ mux { -+ pins = "gpio57", "gpio58"; -+ function = "blsp4_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+&soc { -+ extcon_usb: extcon_usb { -+ pinctrl-0 = <&extcon_usb_pins>; -+ pinctrl-names = "default"; -+ id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; -+ status = "ok"; -+ }; -+ -+ mdio: mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; -+ status = "ok"; -+ ethernet-phy@3 { -+ reg = <0x03>; -+ }; -+ -+ ethernet-phy@4 { -+ reg = <0x18>; -+ }; -+ -+ ethernet-phy@1 { -+ reg = <0x01>; -+ }; -+ -+ ethernet-phy@2 { -+ reg = <0x02>; -+ }; -+ -+ ethernet-phy@0 { -+ reg = <0x00>; -+ }; -+ }; -+ -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <1>; -+ reg = <0x3a001000 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <2>; -+ reg = <0x3a001200 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <1>; -+ phy-mode = "sgmii"; -+ }; -+ -+ ess-switch@3a000000 { -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x4>; /* lan port bitmap */ -+ switch_wan_bmp = <0x2>; /* wan port bitmap */ -+ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -+ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -+ qcom,port_phyinfo { -+ port@1 { -+ phy_address = <0x01>; -+ port_id = <0x02>; -+ }; -+ -+ port@0 { -+ phy_address = <0x00>; -+ port_id = <0x01>; -+ }; -+ -+ port@2 { -+ phy_address = <0x02>; -+ port_id = <0x03>; -+ }; -+ -+ port@3 { -+ phy_address = <0x03>; -+ port_id = <0x04>; -+ }; -+ -+ port@4 { -+ phy_address = <0x18>; -+ port_id = <0x05>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ }; -+ }; -+ -+ gpio_keys { -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ pinctrl-names = "default"; -+ -+ reset { -+ label = "reset"; -+ linux,code = ; -+ gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ }; -+ -+ /* wps { -+ label = "wps"; -+ linux,code = <>; -+ gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ };*/ -+ }; -+ -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ -+ led@25 { -+ label = "green:wifi5"; -+ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:5g"; -+ default-state = "off"; -+ }; -+ led@24 { -+ label = "green:wifi2"; -+ gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:2g"; -+ default-state = "off"; -+ }; -+ led_power: led@16 { -+ label = "green:led_pwr"; -+ gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:power"; -+ default-state = "off"; -+ }; -+ }; -+}; -+ -+&blsp1_uart3 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+ -+&blsp1_uart2 { -+ pinctrl-0 = <&uart2_pins>; -+ pinctrl-names = "default"; -+ dmas = <&blsp_dma 2>, -+ <&blsp_dma 3>; -+ dma-names = "tx", "rx"; -+ status = "ok"; -+}; -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_1 { -+ status = "ok"; -+}; -+ -+&usb2 { -+ status = "ok"; -+}; -+ -+&usb3 { -+ status = "ok"; -+}; -+ -+&nss_crypto { -+ status = "ok"; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts -new file mode 100755 -index 0000000000..d04cb1020c ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap102.dts -@@ -0,0 +1,918 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "qcom-ipq807x-soc.dtsi" -+#include "qcom-ipq807x-ac-cpu.dtsi" -+ -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ model = "Edgecore EAP102"; -+ compatible = "edgecore,eap102", "qcom,ipq807x-ac02", "qcom,ipq807x"; -+ qcom,msm-id = <0x178 0x0>; -+ interrupt-parent = <&intc>; -+ qcom,board-id = <0x8 0x0>; -+ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -+ -+ aliases { -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet1 = "/soc/dp5"; -+ ethernet0 = "/soc/dp6"; -+ -+ led-boot = &led_power; -+ led-failsafe = &led_power; -+ led-running = &led_power; -+ led-upgrade = &led_power; -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ -+ init=/init"; -+ #ifdef __IPQ_MEM_PROFILE_256_MB__ -+ bootargs-append = " swiotlb=1"; -+ #else -+ bootargs-append = " swiotlb=1 coherent_pool=2M"; -+ #endif -+ }; -+}; -+ -+&tlmm { -+ pinctrl-0 = <&btcoex_pins>; -+ pinctrl-names = "default"; -+ -+ btcoex_pins: btcoex_pins { -+ mux_0 { -+ pins = "gpio64"; -+ function = "pta1_1"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "pta1_2"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio68"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio69"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_2 { -+ pins = "gpio33"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ mux_3 { -+ pins = "gpio44"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ }; -+ -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio23", "gpio24"; -+ function = "blsp4_uart1"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ hsuart_pins: hsuart_pins { -+ mux { -+ pins = "gpio49"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ -+ reset_button { -+ pins = "gpio66"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ led_pins: led_pins { -+ led_pwr { -+ pins = "gpio46"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ -+ led_2g { -+ pins = "gpio47"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ -+ led_5g { -+ pins = "gpio48"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ -+ led_bt { -+ pins = "gpio50"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ usb_mux_sel_pins: usb_mux_pins { -+ mux { -+ pins = "gpio27"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ pcie0_pins: pcie_pins { -+ pcie0_rst { -+ pins = "gpio58"; -+ function = "pcie0_rst"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ pcie0_wake { -+ pins = "gpio59"; -+ function = "pcie0_wake"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+}; -+ -+&soc { -+ gpio_keys { -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ pinctrl-names = "default"; -+ -+ button@1 { -+ label = "reset_button"; -+ linux,code = ; -+ gpios = <&tlmm 66 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ }; -+ }; -+ -+ mdio: mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1 &tlmm 44 1>; -+ compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; -+ phy0: ethernet-phy@0 { -+ reg = <0>; -+ }; -+ phy1: ethernet-phy@1 { -+ reg = <1>; -+ }; -+ phy2: ethernet-phy@2 { -+ reg = <2>; -+ }; -+ phy3: ethernet-phy@3 { -+ reg = <3>; -+ }; -+ phy4: ethernet-phy@4 { -+ reg = <24>; -+ }; -+ phy5: ethernet-phy@5 { -+ reg = <28>; -+ }; -+ }; -+ -+ ess-switch@3a000000 { -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x3e>; /* lan port bitmap */ -+ switch_wan_bmp = <0x40>; /* wan port bitmap */ -+ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ -+ bm_tick_mode = <0>; /* bm tick mode */ -+ tm_tick_mode = <0>; /* tm tick mode */ -+ qcom,port_phyinfo { -+ port@0 { -+ port_id = <1>; -+ phy_address = <0>; -+ }; -+ port@1 { -+ port_id = <2>; -+ phy_address = <1>; -+ }; -+ port@2 { -+ port_id = <3>; -+ phy_address = <2>; -+ }; -+ port@3 { -+ port_id = <4>; -+ phy_address = <3>; -+ }; -+ port@4 { -+ port_id = <5>; -+ phy_address = <24>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ port@5 { -+ port_id = <6>; -+ phy_address = <28>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ }; -+ port_scheduler_resource { -+ port@0 { -+ port_id = <0>; -+ ucast_queue = <0 143>; -+ mcast_queue = <256 271>; -+ l0sp = <0 35>; -+ l0cdrr = <0 47>; -+ l0edrr = <0 47>; -+ l1cdrr = <0 7>; -+ l1edrr = <0 7>; -+ }; -+ port@1 { -+ port_id = <1>; -+ ucast_queue = <144 159>; -+ mcast_queue = <272 275>; -+ l0sp = <36 39>; -+ l0cdrr = <48 63>; -+ l0edrr = <48 63>; -+ l1cdrr = <8 11>; -+ l1edrr = <8 11>; -+ }; -+ port@2 { -+ port_id = <2>; -+ ucast_queue = <160 175>; -+ mcast_queue = <276 279>; -+ l0sp = <40 43>; -+ l0cdrr = <64 79>; -+ l0edrr = <64 79>; -+ l1cdrr = <12 15>; -+ l1edrr = <12 15>; -+ }; -+ port@3 { -+ port_id = <3>; -+ ucast_queue = <176 191>; -+ mcast_queue = <280 283>; -+ l0sp = <44 47>; -+ l0cdrr = <80 95>; -+ l0edrr = <80 95>; -+ l1cdrr = <16 19>; -+ l1edrr = <16 19>; -+ }; -+ port@4 { -+ port_id = <4>; -+ ucast_queue = <192 207>; -+ mcast_queue = <284 287>; -+ l0sp = <48 51>; -+ l0cdrr = <96 111>; -+ l0edrr = <96 111>; -+ l1cdrr = <20 23>; -+ l1edrr = <20 23>; -+ }; -+ port@5 { -+ port_id = <5>; -+ ucast_queue = <208 223>; -+ mcast_queue = <288 291>; -+ l0sp = <52 55>; -+ l0cdrr = <112 127>; -+ l0edrr = <112 127>; -+ l1cdrr = <24 27>; -+ l1edrr = <24 27>; -+ }; -+ port@6 { -+ port_id = <6>; -+ ucast_queue = <224 239>; -+ mcast_queue = <292 295>; -+ l0sp = <56 59>; -+ l0cdrr = <128 143>; -+ l0edrr = <128 143>; -+ l1cdrr = <28 31>; -+ l1edrr = <28 31>; -+ }; -+ port@7 { -+ port_id = <7>; -+ ucast_queue = <240 255>; -+ mcast_queue = <296 299>; -+ l0sp = <60 63>; -+ l0cdrr = <144 159>; -+ l0edrr = <144 159>; -+ l1cdrr = <32 35>; -+ l1edrr = <32 35>; -+ }; -+ }; -+ port_scheduler_config { -+ port@0 { -+ port_id = <0>; -+ l1scheduler { -+ group@0 { -+ sp = <0 1>; /*L0 SPs*/ -+ /*cpri cdrr epri edrr*/ -+ cfg = <0 0 0 0>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ /*unicast queues*/ -+ ucast_queue = <0 4 8>; -+ /*multicast queues*/ -+ mcast_queue = <256 260>; -+ /*sp cpri cdrr epri edrr*/ -+ cfg = <0 0 0 0 0>; -+ }; -+ group@1 { -+ ucast_queue = <1 5 9>; -+ mcast_queue = <257 261>; -+ cfg = <0 1 1 1 1>; -+ }; -+ group@2 { -+ ucast_queue = <2 6 10>; -+ mcast_queue = <258 262>; -+ cfg = <0 2 2 2 2>; -+ }; -+ group@3 { -+ ucast_queue = <3 7 11>; -+ mcast_queue = <259 263>; -+ cfg = <0 3 3 3 3>; -+ }; -+ }; -+ }; -+ port@1 { -+ port_id = <1>; -+ l1scheduler { -+ group@0 { -+ sp = <36>; -+ cfg = <0 8 0 8>; -+ }; -+ group@1 { -+ sp = <37>; -+ cfg = <1 9 1 9>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <144>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <272>; -+ mcast_loop_pri = <4>; -+ cfg = <36 0 48 0 48>; -+ }; -+ }; -+ }; -+ port@2 { -+ port_id = <2>; -+ l1scheduler { -+ group@0 { -+ sp = <40>; -+ cfg = <0 12 0 12>; -+ }; -+ group@1 { -+ sp = <41>; -+ cfg = <1 13 1 13>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <160>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <276>; -+ mcast_loop_pri = <4>; -+ cfg = <40 0 64 0 64>; -+ }; -+ }; -+ }; -+ port@3 { -+ port_id = <3>; -+ l1scheduler { -+ group@0 { -+ sp = <44>; -+ cfg = <0 16 0 16>; -+ }; -+ group@1 { -+ sp = <45>; -+ cfg = <1 17 1 17>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <176>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <280>; -+ mcast_loop_pri = <4>; -+ cfg = <44 0 80 0 80>; -+ }; -+ }; -+ }; -+ port@4 { -+ port_id = <4>; -+ l1scheduler { -+ group@0 { -+ sp = <48>; -+ cfg = <0 20 0 20>; -+ }; -+ group@1 { -+ sp = <49>; -+ cfg = <1 21 1 21>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <192>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <284>; -+ mcast_loop_pri = <4>; -+ cfg = <48 0 96 0 96>; -+ }; -+ }; -+ }; -+ port@5 { -+ port_id = <5>; -+ l1scheduler { -+ group@0 { -+ sp = <52>; -+ cfg = <0 24 0 24>; -+ }; -+ group@1 { -+ sp = <53>; -+ cfg = <1 25 1 25>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <208>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <288>; -+ mcast_loop_pri = <4>; -+ cfg = <52 0 112 0 112>; -+ }; -+ }; -+ }; -+ port@6 { -+ port_id = <6>; -+ l1scheduler { -+ group@0 { -+ sp = <56>; -+ cfg = <0 28 0 28>; -+ }; -+ group@1 { -+ sp = <57>; -+ cfg = <1 29 1 29>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <224>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <292>; -+ mcast_loop_pri = <4>; -+ cfg = <56 0 128 0 128>; -+ }; -+ }; -+ }; -+ port@7 { -+ port_id = <7>; -+ l1scheduler { -+ group@0 { -+ sp = <60>; -+ cfg = <0 32 0 32>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <240>; -+ mcast_queue = <296>; -+ cfg = <60 0 144 0 144>; -+ }; -+ }; -+ }; -+ }; -+ }; -+/* -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <1>; -+ reg = <0x3a001000 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <2>; -+ reg = <0x3a001200 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <1>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp3 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <3>; -+ reg = <0x3a001400 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <2>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp4 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <4>; -+ reg = <0x3a001600 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <3>; -+ phy-mode = "sgmii"; -+ }; -+*/ -+ dp6 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <6>; -+ reg = <0x3a001800 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <28>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp5 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <5>; -+ reg = <0x3a001a00 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <24>; -+ phy-mode = "sgmii"; -+ }; -+ -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&led_pins>; -+ pinctrl-names = "default"; -+ -+ led_pwr { -+ label = "green:wan"; -+ gpios = <&tlmm 46 GPIO_ACTIVE_HIGH>; -+ default-state = "off"; -+ linux,default-trigger = "led_pwr"; -+ }; -+ -+ led_2g { -+ label = "green:wifi2"; -+ gpio = <&tlmm 47 GPIO_ACTIVE_HIGH>; -+ default-state = "off"; -+ }; -+ -+ led_5g { -+ label = "green:wifi5"; -+ gpio = <&tlmm 48 GPIO_ACTIVE_HIGH>; -+ default-state = "off"; -+ }; -+ -+ led_power: led_bt { -+ gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; -+ label = "green:power"; -+ default-state = "on"; -+ linux,default-trigger = "led_bt"; -+ }; -+ }; -+ nss-macsec0 { -+ compatible = "qcom,nss-macsec"; -+ phy_addr = <0x18>; -+ phy_access_mode = <0>; -+ mdiobus = <&mdio>; -+ }; -+ nss-macsec1 { -+ compatible = "qcom,nss-macsec"; -+ phy_addr = <0x1c>; -+ phy_access_mode = <0>; -+ mdiobus = <&mdio>; -+ }; -+}; -+ -+&serial_blsp4 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { /* BLSP1 QUP1 */ -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+ -+&serial_blsp2 { -+ pinctrl-0 = <&hsuart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&nss0 { -+ qcom,low-frequency = <187200000>; -+ qcom,mid-frequency = <748800000>; -+ qcom,max-frequency = <1497600000>; -+}; -+ -+&msm_imem { -+ status = "disabled"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&ssphy_1 { -+ status = "ok"; -+}; -+ -+&qusb_phy_1 { -+ status = "ok"; -+}; -+ -+&usb3_0 { -+ status = "ok"; -+}; -+ -+&usb3_1 { -+ status = "ok"; -+}; -+ -+&cryptobam { -+ status = "ok"; -+}; -+ -+&crypto { -+ status = "ok"; -+}; -+ -+&i2c_0 { -+ status = "disabled"; -+}; -+ -+&i2c_1 { -+ status = "disabled"; -+}; -+ -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&qpic_lcd { -+ status = "disabled"; -+}; -+ -+&qpic_lcd_panel { -+ status = "disabled"; -+}; -+ -+&ledc { -+ status = "disabled"; -+}; -+ -+&pcie0 { -+ status = "ok"; -+}; -+ -+&pcie1 { -+ status = "disabled"; -+}; -+ -+&glink_rpm { -+ status = "disabled"; -+}; -+ -+&apc_cpr { -+ /* Same CPR configuration as OAK */ -+ compatible = "qcom,cpr4-ipq817x-apss-regulator"; -+ -+ thread@0 { -+ apc_vreg: regulator { -+ regulator-min-microvolt = <1>; -+ regulator-max-microvolt = <2>; -+ qcom,cpr-fuse-corners = <2>; -+ qcom,cpr-corners = <3>; -+ qcom,cpr-speed-bin-corners = <3>; -+ qcom,cpr-corner-fmax-map = <1 3>; -+ -+ qcom,cpr-voltage-ceiling = -+ <840000 904000 944000>; -+ qcom,cpr-voltage-floor = -+ <592000 648000 712000>; -+ qcom,corner-frequencies = -+ <1017600000 1382400000 1382400000>; -+ -+ qcom,cpr-open-loop-voltage-fuse-adjustment-0 = -+ /* Speed bin 0; CPR rev 0..7 */ -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>; -+ -+ qcom,cpr-open-loop-voltage-fuse-adjustment-1 = -+ /* Speed bin 0; CPR rev 0..7 */ -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 20000 26000>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>; -+ -+ qcom,cpr-open-loop-voltage-fuse-adjustment-v2-0 = -+ /* Speed bin 0; CPR rev 0..7 */ -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>; -+ -+ qcom,cpr-open-loop-voltage-fuse-adjustment-v2-1 = -+ /* Speed bin 0; CPR rev 0..7 */ -+ < 0 0>, -+ < 0 7000>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>, -+ < 0 0>; -+ -+ qcom,cpr-floor-to-ceiling-max-range = -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>, -+ < 40000 40000 40000>; -+ }; -+ }; -+}; -+ -+&npu_cpr { -+ status = "disabled"; -+}; -+ -+&nss0 { -+ npu-supply = <&dummy_reg>; -+ mx-supply = <&dummy_reg>; -+}; -+ -+&wifi0 { -+ qcom,board_id = <0x92>; -+}; -+ -+&wifi1 { -+ qcom,board_id = <0x292>; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts -new file mode 100644 -index 0000000000..4c633fea5c ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c4.dts -@@ -0,0 +1,942 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+#include "qcom-ipq807x-soc.dtsi" -+#include "qcom-ipq807x-hk-cpu.dtsi" -+ -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ model = "CIG WF194c4"; -+ compatible = "cig,wf194c4", "qcom,ipq807x"; -+ qcom,msm-id = <0x156 0x0>; -+ interrupt-parent = <&intc>; -+ qcom,board-id = <0x8 0x0>; -+ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -+ -+ aliases { -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet0 = "/soc/dp1"; -+ ethernet1 = "/soc/dp2"; -+ /* ethernet2 = "/soc/dp3"; -+ ethernet3 = "/soc/dp4"; -+ ethernet4 = "/soc/dp5"; -+ ethernet5 = "/soc/dp6"; -+ */ -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ -+ init=/init"; -+ #ifdef __IPQ_MEM_PROFILE_256_MB__ -+ bootargs-append = " swiotlb=1"; -+ #else -+ bootargs-append = " swiotlb=1 coherent_pool=2M"; -+ #endif -+ }; -+}; -+ -+&tlmm { -+ pinctrl-0 = <&btcoex_pins>; -+ pinctrl-names = "default"; -+ -+ btcoex_pins: btcoex_pins { -+/* -+ mux_0 { -+ pins = "gpio64"; -+ function = "pta1_1"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "pta1_2"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_2 { -+ pins = "gpio66"; -+ function = "pta1_0"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_3 { -+ pins = "gpio54"; -+ function = "pta2_0"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_4 { -+ pins = "gpio55"; -+ function = "pta2_1"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+ mux_5 { -+ pins = "gpio56"; -+ function = "pta2_2"; -+ drive-strength = <6>; -+ bias-pull-down; -+ }; -+*/ -+ mux_0 { -+ pins = "gpio34"; -+ function = "gpio"; -+ drive-strength = <6>; -+ bias-pull-up; -+ output-high; -+ }; -+ mux_1 { -+ pins = "gpio62"; -+ function = "gpio"; -+ drive-strength = <6>; -+ bias-pull-up; -+ output-high; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio68"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio69"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio23", "gpio24"; -+ function = "blsp4_uart1"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ /*spi_3_pins: spi_3_pins { -+ mux { -+ pins = "gpio50", "gpio52", "gpio53"; -+ function = "blsp3_spi"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ spi_cs { -+ pins = "gpio22"; -+ function = "blsp3_spi2"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ quartz_interrupt { -+ pins = "gpio47"; -+ function = "gpio"; -+ input; -+ bias-disable; -+ }; -+ quartz_reset { -+ pins = "gpio21"; -+ function = "gpio"; -+ output-low; -+ bias-disable; -+ }; -+ };*/ -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ hsuart_pins: hsuart_pins { -+ mux { -+ pins = "gpio48", "gpio49"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ wps_button { -+ pins = "gpio67"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ leds_pins: leds_pinmux { -+ led1_r { -+ pins = "gpio54"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led1_g { -+ pins = "gpio55"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led2_r { -+ pins = "gpio56"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led2_g { -+ pins = "gpio64"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ -+ }; -+ -+ /*usb_mux_sel_pins: usb_mux_pins { -+ mux { -+ pins = "gpio27"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ pcie0_pins: pcie_pins { -+ pcie0_rst { -+ pins = "gpio58"; -+ function = "pcie0_rst"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ pcie0_wake { -+ pins = "gpio59"; -+ function = "pcie0_wake"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ };*/ -+ uniphy_pins: uniphy_pinmux { -+ mux_2 { -+ pins = "gpio37"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_3 { -+ pins = "gpio44"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+}; -+ -+&soc { -+ gpio_keys { -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ pinctrl-names = "default"; -+ -+ button@1 { -+ label = "reset"; -+ linux,code = ; -+ gpios = <&tlmm 67 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ }; -+ }; -+ -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+ -+ led@54 { -+ label = "red:lan"; -+ gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "led1_r"; -+ default-state = "off"; -+ }; -+ led@55 { -+ label = "green:lan"; -+ gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "led1_g"; -+ default-state = "off"; -+ }; -+ led@56 { -+ label = "red:wan"; -+ gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "led2_r"; -+ default-state = "off"; -+ }; -+ led@64 { -+ label = "green:wan"; -+ gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "led2_g"; -+ default-state = "off"; -+ }; -+ }; -+ mdio: mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 37 0 &tlmm 44 0>; -+ phy0: ethernet-phy@0 { -+ reg = <0x10>; /*<0>*/ -+ }; -+ phy1: ethernet-phy@1 { -+ reg = <0x11>; -+ }; -+ phy2: ethernet-phy@2 { -+ reg = <0x12>; -+ }; -+ phy3: ethernet-phy@3 { -+ reg = <0x13>; -+ }; -+ phy4: ethernet-phy@4 { -+ reg = <0x14>; -+ }; -+ phy5: ethernet-phy@5 { -+ compatible ="ethernet-phy-ieee802.3-c45"; -+ reg = <0>; -+ }; -+ }; -+ -+ ess-switch@3a000000 { -+ pinctrl-0 = <&uniphy_pins>; -+ pinctrl-names = "default"; -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x32>; /*..0x3e lan port bitmap */ -+ switch_wan_bmp = <0x40>; /* wan port bitmap */ -+ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ -+ bm_tick_mode = <0>; /* bm tick mode */ -+ tm_tick_mode = <0>; /* tm tick mode */ -+ /*qcom,port_phyinfo { -+ port@0 { -+ port_id = <1>; -+ phy_address = <0>; -+ }; -+ port@1 { -+ port_id = <2>; -+ phy_address = <1>; -+ }; -+ port@2 { -+ port_id = <3>; -+ phy_address = <2>; -+ }; -+ port@3 { -+ port_id = <4>; -+ phy_address = <3>; -+ }; -+ port@4 { -+ port_id = <5>; -+ phy_address = <24>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ port@5 { -+ port_id = <6>; -+ phy_address = <28>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ };*/ -+ port_scheduler_resource { -+ port@0 { -+ port_id = <0>; -+ ucast_queue = <0 143>; -+ mcast_queue = <256 271>; -+ l0sp = <0 35>; -+ l0cdrr = <0 47>; -+ l0edrr = <0 47>; -+ l1cdrr = <0 7>; -+ l1edrr = <0 7>; -+ }; -+ port@1 { -+ port_id = <1>; -+ ucast_queue = <144 159>; -+ mcast_queue = <272 275>; -+ l0sp = <36 39>; -+ l0cdrr = <48 63>; -+ l0edrr = <48 63>; -+ l1cdrr = <8 11>; -+ l1edrr = <8 11>; -+ }; -+ port@2 { -+ port_id = <2>; -+ ucast_queue = <160 175>; -+ mcast_queue = <276 279>; -+ l0sp = <40 43>; -+ l0cdrr = <64 79>; -+ l0edrr = <64 79>; -+ l1cdrr = <12 15>; -+ l1edrr = <12 15>; -+ }; -+ port@3 { -+ port_id = <3>; -+ ucast_queue = <176 191>; -+ mcast_queue = <280 283>; -+ l0sp = <44 47>; -+ l0cdrr = <80 95>; -+ l0edrr = <80 95>; -+ l1cdrr = <16 19>; -+ l1edrr = <16 19>; -+ }; -+ port@4 { -+ port_id = <4>; -+ ucast_queue = <192 207>; -+ mcast_queue = <284 287>; -+ l0sp = <48 51>; -+ l0cdrr = <96 111>; -+ l0edrr = <96 111>; -+ l1cdrr = <20 23>; -+ l1edrr = <20 23>; -+ }; -+ port@5 { -+ port_id = <5>; -+ ucast_queue = <208 223>; -+ mcast_queue = <288 291>; -+ l0sp = <52 55>; -+ l0cdrr = <112 127>; -+ l0edrr = <112 127>; -+ l1cdrr = <24 27>; -+ l1edrr = <24 27>; -+ }; -+ port@6 { -+ port_id = <6>; -+ ucast_queue = <224 239>; -+ mcast_queue = <292 295>; -+ l0sp = <56 59>; -+ l0cdrr = <128 143>; -+ l0edrr = <128 143>; -+ l1cdrr = <28 31>; -+ l1edrr = <28 31>; -+ }; -+ port@7 { -+ port_id = <7>; -+ ucast_queue = <240 255>; -+ mcast_queue = <296 299>; -+ l0sp = <60 63>; -+ l0cdrr = <144 159>; -+ l0edrr = <144 159>; -+ l1cdrr = <32 35>; -+ l1edrr = <32 35>; -+ }; -+ }; -+ port_scheduler_config { -+ port@0 { -+ port_id = <0>; -+ l1scheduler { -+ group@0 { -+ sp = <0 1>; /*L0 SPs*/ -+ /*cpri cdrr epri edrr*/ -+ cfg = <0 0 0 0>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ /*unicast queues*/ -+ ucast_queue = <0 4 8>; -+ /*multicast queues*/ -+ mcast_queue = <256 260>; -+ /*sp cpri cdrr epri edrr*/ -+ cfg = <0 0 0 0 0>; -+ }; -+ group@1 { -+ ucast_queue = <1 5 9>; -+ mcast_queue = <257 261>; -+ cfg = <0 1 1 1 1>; -+ }; -+ group@2 { -+ ucast_queue = <2 6 10>; -+ mcast_queue = <258 262>; -+ cfg = <0 2 2 2 2>; -+ }; -+ group@3 { -+ ucast_queue = <3 7 11>; -+ mcast_queue = <259 263>; -+ cfg = <0 3 3 3 3>; -+ }; -+ }; -+ }; -+ port@1 { -+ port_id = <1>; -+ l1scheduler { -+ group@0 { -+ sp = <36>; -+ cfg = <0 8 0 8>; -+ }; -+ group@1 { -+ sp = <37>; -+ cfg = <1 9 1 9>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <144>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <272>; -+ mcast_loop_pri = <4>; -+ cfg = <36 0 48 0 48>; -+ }; -+ }; -+ }; -+ port@2 { -+ port_id = <2>; -+ l1scheduler { -+ group@0 { -+ sp = <40>; -+ cfg = <0 12 0 12>; -+ }; -+ group@1 { -+ sp = <41>; -+ cfg = <1 13 1 13>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <160>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <276>; -+ mcast_loop_pri = <4>; -+ cfg = <40 0 64 0 64>; -+ }; -+ }; -+ }; -+ port@3 { -+ port_id = <3>; -+ l1scheduler { -+ group@0 { -+ sp = <44>; -+ cfg = <0 16 0 16>; -+ }; -+ group@1 { -+ sp = <45>; -+ cfg = <1 17 1 17>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <176>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <280>; -+ mcast_loop_pri = <4>; -+ cfg = <44 0 80 0 80>; -+ }; -+ }; -+ }; -+ port@4 { -+ port_id = <4>; -+ l1scheduler { -+ group@0 { -+ sp = <48>; -+ cfg = <0 20 0 20>; -+ }; -+ group@1 { -+ sp = <49>; -+ cfg = <1 21 1 21>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <192>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <284>; -+ mcast_loop_pri = <4>; -+ cfg = <48 0 96 0 96>; -+ }; -+ }; -+ }; -+ port@5 { -+ port_id = <5>; -+ l1scheduler { -+ group@0 { -+ sp = <52>; -+ cfg = <0 24 0 24>; -+ }; -+ group@1 { -+ sp = <53>; -+ cfg = <1 25 1 25>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <208>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <288>; -+ mcast_loop_pri = <4>; -+ cfg = <52 0 112 0 112>; -+ }; -+ }; -+ }; -+ port@6 { -+ port_id = <6>; -+ l1scheduler { -+ group@0 { -+ sp = <56>; -+ cfg = <0 28 0 28>; -+ }; -+ group@1 { -+ sp = <57>; -+ cfg = <1 29 1 29>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <224>; -+ ucast_loop_pri = <16>; -+ mcast_queue = <292>; -+ mcast_loop_pri = <4>; -+ cfg = <56 0 128 0 128>; -+ }; -+ }; -+ }; -+ port@7 { -+ port_id = <7>; -+ l1scheduler { -+ group@0 { -+ sp = <60>; -+ cfg = <0 32 0 32>; -+ }; -+ }; -+ l0scheduler { -+ group@0 { -+ ucast_queue = <240>; -+ mcast_queue = <296>; -+ cfg = <60 0 144 0 144>; -+ }; -+ }; -+ }; -+ }; -+ }; -+/* -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <1>; -+ reg = <0x3a001000 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <2>; -+ reg = <0x3a001200 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <1>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp3 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <3>; -+ reg = <0x3a001400 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <2>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp4 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <4>; -+ reg = <0x3a001600 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <3>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp5 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <5>; -+ reg = <0x3a003000 0x3fff>; -+ qcom,mactype = <1>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <24>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp6 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <6>; -+ reg = <0x3a007000 0x3fff>; -+ qcom,mactype = <1>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <28>; -+ phy-mode = "sgmii"; -+ }; -+*/ -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <4>; -+ reg = <0x3a001600 0x200>; -+ // qcom,id = <1>; -+ // reg = <0x3a001000 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0x13>; -+ phy-mode = "sgmii"; -+ }; -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <6>; -+ reg = <0x3a007000 0x3fff>; -+ qcom,mactype = <1>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0>; -+ phy-mode = "sgmii"; -+ }; -+/* -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&led_pins>; -+ pinctrl-names = "default"; -+ -+ led_2g { -+ label = "led_2g"; -+ gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; -+ default-state = "off"; -+ }; -+ -+ led_5g { -+ label = "led_5g"; -+ gpio = <&tlmm 43 GPIO_ACTIVE_HIGH>; -+ default-state = "off"; -+ }; -+ }; -+ -+ nss-macsec0 { -+ compatible = "qcom,nss-macsec"; -+ phy_addr = <0x18>; -+ phy_access_mode = <0>; -+ mdiobus = <&mdio>; -+ }; -+ nss-macsec1 { -+ compatible = "qcom,nss-macsec"; -+ phy_addr = <0x1c>; -+ phy_access_mode = <0>; -+ mdiobus = <&mdio>; -+ }; -+*/ -+}; -+ -+&serial_blsp4 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { /* BLSP1 QUP1 */ -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+/* -+&spi_4 { -+ pinctrl-0 = <&spi_3_pins>; -+ pinctrl-names = "default"; -+ cs-select = <2>; -+ quartz-reset-gpio = <&tlmm 21 1>; -+ status = "ok"; -+ spidev3: spi@3 { -+ compatible = "qca,spidev"; -+ reg = <0>; -+ spi-max-frequency = <24000000>; -+ }; -+};*/ -+ -+&serial_blsp2 { -+ pinctrl-0 = <&hsuart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&msm_imem { -+ status = "disabled"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&ssphy_1 { -+ status = "ok"; -+}; -+ -+&qusb_phy_1 { -+ status = "ok"; -+}; -+ -+&usb3_0 { -+ status = "ok"; -+}; -+ -+&usb3_1 { -+ status = "ok"; -+}; -+ -+&cryptobam { -+ status = "ok"; -+}; -+ -+&crypto { -+ status = "ok"; -+}; -+ -+&i2c_0 { -+ status = "disabled"; -+}; -+ -+&i2c_1 { -+ status = "disabled"; -+}; -+ -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&qpic_lcd { -+ status = "disabled"; -+}; -+ -+&qpic_lcd_panel { -+ status = "disabled"; -+}; -+ -+&ledc { -+ status = "disabled"; -+}; -+ -+&pcie0 { -+ status = "disabled"; -+}; -+ -+&pcie1 { -+ status = "disabled"; -+}; -+ -diff --git a/target/linux/ipq807x/image/Makefile b/target/linux/ipq807x/image/Makefile -new file mode 100644 -index 0000000000..3a219e6d6c ---- /dev/null -+++ b/target/linux/ipq807x/image/Makefile -@@ -0,0 +1,26 @@ -+include $(TOPDIR)/rules.mk -+include $(INCLUDE_DIR)/image.mk -+ -+IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(IMG_PREFIX_VERNUM)$(IMG_PREFIX_VERCODE)$(IMG_PREFIX_EXTRA)$(BOARD) -+ -+# default all platform image(fit) build -+define Device/Default -+ PROFILES = Default $$(DEVICE_NAME) -+ KERNEL_NAME := zImage -+ FILESYSTEMS := squashfs -+ DEVICE_DTS_DIR := $(DTS_DIR) -+ KERNEL_IN_UBI := 1 -+ ROOTFSNAME_IN_UBI := ubi_rootfs -+ BLOCKSIZE := 128k -+ PAGESIZE := 2048 -+ IMAGES := sysupgrade.tar nand-factory.bin -+ IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata -+ IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand -+ KERNEL_NAME := Image -+ KERNEL = kernel-bin | gzip | fit gzip $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb -+ KERNEL_INITRAMFS = kernel-bin | gzip | fit gzip $$(KDIR)/image-$$(firstword $$(DEVICE_DTS)).dtb -+endef -+ -+include $(SUBTARGET).mk -+ -+$(eval $(call BuildImage)) -diff --git a/target/linux/ipq807x/image/ipq50xx.mk b/target/linux/ipq807x/image/ipq50xx.mk -new file mode 100644 -index 0000000000..f20d54190e ---- /dev/null -+++ b/target/linux/ipq807x/image/ipq50xx.mk -@@ -0,0 +1,10 @@ -+KERNEL_LOADADDR := 0x41208000 -+ -+define Device/qcom_mp03_3 -+ DEVICE_TITLE := Qualcomm Maple 03.3 -+ DEVICE_DTS := qcom-ipq5018-mp03.3 -+ SUPPORTED_DEVICES := qcom,ipq5018-mp03.3 -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq5018 -+ DEVICE_DTS_CONFIG := config@mp03.3 -+endef -+TARGET_DEVICES += qcom_mp03_3 -diff --git a/target/linux/ipq807x/image/ipq60xx.mk b/target/linux/ipq807x/image/ipq60xx.mk -new file mode 100644 -index 0000000000..e94ab22785 ---- /dev/null -+++ b/target/linux/ipq807x/image/ipq60xx.mk -@@ -0,0 +1,56 @@ -+KERNEL_LOADADDR := 0x41008000 -+ -+define Device/cig_wf188 -+ DEVICE_TITLE := Cigtech WF-188 -+ DEVICE_DTS := qcom-ipq6018-cig-wf188 -+ DEVICE_DTS_CONFIG := config@cp03-c1 -+ SUPPORTED_DEVICES := cig,wf188 -+ IMAGES := sysupgrade.tar -+ IMAGE/sysupgrade.tar/squashfs := append-rootfs | pad-rootfs | sysupgrade-tar rootfs=$$$$@ | append-metadata -+ DEVICE_PACKAGES := ath11k-wifi-cig-wf188 uboot-env -+endef -+TARGET_DEVICES += cig_wf188 -+ -+define Device/cig_wf188n -+ DEVICE_TITLE := Cigtech WF-188n -+ DEVICE_DTS := qcom-ipq6018-cig-wf188n -+ DEVICE_DTS_CONFIG := config@cp03-c1 -+ SUPPORTED_DEVICES := cig,wf188n -+ DEVICE_PACKAGES := ath11k-wifi-cig-wf188n uboot-env -+endef -+TARGET_DEVICES += cig_wf188n -+ -+define Device/edgecore_eap101 -+ DEVICE_TITLE := EdgeCore EAP101 -+ DEVICE_DTS := qcom-ipq6018-edgecore-eap101 -+ DEVICE_DTS_CONFIG := config@cp01-c1 -+ SUPPORTED_DEVICES := edgecore,eap101 -+ DEVICE_PACKAGES := ath11k-wifi-edgecore-eap101 uboot-envtools -+endef -+TARGET_DEVICES += edgecore_eap101 -+ -+define Device/wallys_dr6018 -+ DEVICE_TITLE := Wallys DR6018 -+ DEVICE_DTS := qcom-ipq6018-wallys-dr6018 -+ DEVICE_DTS_CONFIG := config@cp01-c4 -+ SUPPORTED_DEVICES := wallys,dr6018 -+ DEVICE_PACKAGES := ath11k-wifi-wallys-dr6018 uboot-envtools -+endef -+TARGET_DEVICES += wallys_dr6018 -+ -+define Device/qcom_cp01_c1 -+ DEVICE_TITLE := Qualcomm Cypress C1 -+ DEVICE_DTS := qcom-ipq6018-cp01-c1 -+ SUPPORTED_DEVICES := qcom,ipq6018-cp01 -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 -+endef -+TARGET_DEVICES += qcom_cp01_c1 -+ -+define Device/xiaomi_ax1800 -+ DEVICE_TITLE := Xiaomi AX1800 -+ DEVICE_DTS := qcom-ipq6018-miwifi-ax1800 -+ SUPPORTED_DEVICES := xiaomi,ax1800 -+ DEVICE_DTS_CONFIG := config@cp03-c1 -+ DEVICE_PACKAGES := ath11k-wifi-xiaomi-ax1800 -+endef -+TARGET_DEVICES += xiaomi_ax1800 -diff --git a/target/linux/ipq807x/image/ipq807x.mk b/target/linux/ipq807x/image/ipq807x.mk -new file mode 100644 -index 0000000000..ba39ad8466 ---- /dev/null -+++ b/target/linux/ipq807x/image/ipq807x.mk -@@ -0,0 +1,90 @@ -+KERNEL_LOADADDR := 0x41208000 -+ -+define Device/qcom_hk01 -+ DEVICE_TITLE := Qualcomm Hawkeye HK01 -+ DEVICE_DTS := qcom-ipq807x-hk01 -+ DEVICE_DTS_CONFIG=config@hk01 -+ SUPPORTED_DEVICES := qcom,ipq807x-hk01 -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq8074 -+endef -+TARGET_DEVICES += qcom_hk01 -+ -+define Device/qcom_hk14 -+ DEVICE_TITLE := Qualcomm Hawkeye HK14 -+ DEVICE_DTS := qcom-ipq807x-hk14 -+ DEVICE_DTS_CONFIG=config@hk14 -+ SUPPORTED_DEVICES := qcom,ipq807x-hk14 -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq8074 kmod-ath11k-pci ath11k-firmware-qcn9000 -+endef -+TARGET_DEVICES += qcom_hk14 -+ -+define Device/sercomm_wallaby -+ DEVICE_TITLE := Sercomm Kiwi -+ DEVICE_DTS := qcom-ipq807x-sercomm-wallaby -+ DEVICE_DTS_CONFIG=config@hk09 -+ SUPPORTED_DEVICES := sercomm,wallaby -+ DEVICE_PACKAGES := ath11k-wifi-sercomm-wallaby -+endef -+TARGET_DEVICES += sercomm_wallaby -+ -+define Device/cig_wf194 -+ DEVICE_TITLE := CIG WF194C -+ DEVICE_DTS := qcom-ipq807x-wf194c -+ DEVICE_DTS_CONFIG=config@hk01 -+ SUPPORTED_DEVICES := cig,wf194c -+ DEVICE_PACKAGES := ath11k-wifi-cig-wf194c aq-fw-download uboot-envtools kmod-usb3 kmod-usb2 -+endef -+TARGET_DEVICES += cig_wf194 -+ -+define Device/cig_wf194c4 -+ DEVICE_TITLE := CIG WF194C4 -+ DEVICE_DTS := qcom-ipq807x-wf194c4 -+ DEVICE_DTS_CONFIG=config@hk09 -+ SUPPORTED_DEVICES := cig,wf194c4 -+ DEVICE_PACKAGES := ath11k-wifi-cig-wf194c4 aq-fw-download uboot-envtools kmod-usb3 kmod-usb2 -+endef -+TARGET_DEVICES += cig_wf194c4 -+ -+define Device/edgecore_eap102 -+ DEVICE_TITLE := Edgecore EAP102 -+ DEVICE_DTS := qcom-ipq807x-eap102 -+ DEVICE_DTS_CONFIG=config@ac02 -+ SUPPORTED_DEVICES := edgecore,eap102 -+ DEVICE_PACKAGES := ath11k-wifi-edgecore-eap102 kmod-usb3 uboot-envtools -+endef -+TARGET_DEVICES += edgecore_eap102 -+ -+define Device/edgecore_eap106 -+ DEVICE_TITLE := Edgecore EAP102 -+ DEVICE_DTS := qcom-ipq807x-eap106 -+ DEVICE_DTS_CONFIG=config@hk02 -+ SUPPORTED_DEVICES := edgecore,eap106 -+ DEVICE_PACKAGES := ath11k-wifi-edgecore-eap106 aq-fw-download kmod-usb2 kmod-usb3 uboot-envtools -+endef -+TARGET_DEVICES += edgecore_eap106 -+ -+define Device/tplink_ex227 -+ DEVICE_TITLE := TP-Link EX227 -+ DEVICE_DTS := qcom-ipq807x-ex227 -+ DEVICE_DTS_CONFIG=config@hk07 -+ SUPPORTED_DEVICES := tplink,ex227 -+ DEVICE_PACKAGES := ath11k-wifi-tplink-ex227 -+ IMAGES := sysupgrade.tar nand-factory.bin nand-factory.ubi -+ IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata -+ IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand -+ IMAGE/nand-factory.ubi := append-ubi -+endef -+TARGET_DEVICES += tplink_ex227 -+ -+define Device/tplink_ex447 -+ DEVICE_TITLE := TP-Link EX447 -+ DEVICE_DTS := qcom-ipq807x-ex447 -+ DEVICE_DTS_CONFIG=config@hk09 -+ SUPPORTED_DEVICES := tplink,ex447 -+ DEVICE_PACKAGES := ath11k-wifi-tplink-ex447 -+ IMAGES := sysupgrade.tar nand-factory.bin nand-factory.ubi -+ IMAGE/sysupgrade.tar := sysupgrade-tar | append-metadata -+ IMAGE/nand-factory.bin := append-ubi | qsdk-ipq-factory-nand -+ IMAGE/nand-factory.ubi := append-ubi -+endef -+TARGET_DEVICES += tplink_ex447 -diff --git a/target/linux/ipq807x/ipq50xx/config-default b/target/linux/ipq807x/ipq50xx/config-default -new file mode 100644 -index 0000000000..b8e202c874 ---- /dev/null -+++ b/target/linux/ipq807x/ipq50xx/config-default -@@ -0,0 +1,84 @@ -+# CONFIG_AHCI_IPQ is not set -+CONFIG_ARCH_IPQ5018=y -+# CONFIG_DIAGFWD_BRIDGE_CODE is not set -+CONFIG_IPQ_ADSS_5018=y -+CONFIG_IPQ_APSS_5018=y -+CONFIG_IPQ_GCC_5018=y -+# CONFIG_NET_SWITCHDEV is not set -+CONFIG_NUM_ALT_PARTITION=16 -+CONFIG_PINCTRL_IPQ5018=y -+# CONFIG_IPC_LOGGING is not set -+CONFIG_IPQ_SUBSYSTEM_DUMP=y -+CONFIG_SPS=y -+CONFIG_SPS_SUPPORT_NDP_BAM=y -+CONFIG_CORESIGHT=y -+CONFIG_CORESIGHT_CSR=y -+CONFIG_CORESIGHT_CTI=y -+CONFIG_CORESIGHT_EVENT=y -+CONFIG_CORESIGHT_HWEVENT=y -+CONFIG_CORESIGHT_LINKS_AND_SINKS=y -+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y -+CONFIG_CORESIGHT_QCOM_REPLICATOR=y -+# CONFIG_INPUT_PM8941_PWRKEY is not set -+CONFIG_MDIO_QCA=y -+# CONFIG_CRYPTO_ALL_CASES is not set -+CONFIG_CRYPTO_DEV_QCOM_ICE=y -+# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set -+CONFIG_CRYPTO_SHA512=y -+# CONFIG_CORESIGHT_QPDI is not set -+# CONFIG_CORESIGHT_SINK_ETBV10 is not set -+CONFIG_CORESIGHT_SINK_TPIU=y -+# CONFIG_CORESIGHT_SOURCE_DUMMY is not set -+CONFIG_CORESIGHT_SOURCE_ETM3X=y -+CONFIG_CORESIGHT_SOURCE_ETM4X=y -+# CONFIG_CORESIGHT_REMOTE_ETM is not set -+CONFIG_CORESIGHT_STM=y -+CONFIG_CORESIGHT_TPDA=y -+CONFIG_CORESIGHT_TPDM=y -+# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set -+CONFIG_IIO=y -+# CONFIG_IIO_BUFFER is not set -+# CONFIG_IIO_TRIGGER is not set -+CONFIG_PCIE_DW_PLAT=y -+CONFIG_PHY_IPQ_UNIPHY_PCIE=y -+CONFIG_VMSPLIT_2G=y -+# CONFIG_VMSPLIT_3G is not set -+CONFIG_PPS=y -+CONFIG_PTP_1588_CLOCK=y -+# CONFIG_DP83640_PHY is not set -+CONFIG_PWM_IPQ5018=y -+CONFIG_QCOM_APM=y -+CONFIG_QCOM_DCC=y -+# CONFIG_QCOM_SPMI_TEMP_ALARM is not set -+CONFIG_MMC_SDHCI_MSM_ICE=y -+CONFIG_USB_BAM=y -+CONFIG_MAILBOX=y -+# CONFIG_USB_QCOM_DIAG_BRIDGE is not set -+# CONFIG_USB_CONFIGFS_F_DIAG is not set -+# CONFIG_NF_IPV6_DUMMY_HEADER is not set -+CONFIG_RMNET_DATA=y -+CONFIG_RMNET_DATA_DEBUG_PKT=y -+CONFIG_MTD_NAND_SERIAL=y -+CONFIG_PAGE_SCOPE_MULTI_PAGE_READ=y -+# CONFIG_RMNET_DATA_FC is not set -+CONFIG_CRYPTO_NO_ZERO_LEN_HASH=y -+CONFIG_CRYPTO_DISABLE_AES192_TEST=y -+# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set -+# CONFIG_CPU_FREQ_GOV_PERFORMANCE is not set -+# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set -+# CONFIG_CPU_FREQ_GOV_ONDEMAND is not set -+# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set -+CONFIG_QTI_EUD=y -+CONFIG_USB_QCA_M31_PHY=y -+CONFIG_QGIC2_MSI=y -+CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y -+CONFIG_PWM_IPQ4019=y -+CONFIG_RMNET=y -+CONFIG_QCOM_QMI_RMNET=y -+CONFIG_QCOM_QMI_DFC=y -+CONFIG_QCOM_QMI_POWER_COLLAPSE=y -+CONFIG_RMNET_CTL=y -+CONFIG_RMNET_CTL_DEBUG=y -+CONFIG_SND_SOC_IPQ_LPASS=y -+CONFIG_SND_SOC_IPQ_LPASS_PCM_RAW=y -+# CONFIG_SND_SOC_IPQ_PCM_RAW is not set -diff --git a/target/linux/ipq807x/ipq50xx/config-lowmem b/target/linux/ipq807x/ipq50xx/config-lowmem -new file mode 100644 -index 0000000000..b1b817ef6f ---- /dev/null -+++ b/target/linux/ipq807x/ipq50xx/config-lowmem -@@ -0,0 +1,73 @@ -+# CONFIG_AHCI_IPQ is not set -+CONFIG_ARCH_IPQ5018=y -+# CONFIG_DIAGFWD_BRIDGE_CODE is not set -+CONFIG_IPQ_ADSS_5018=y -+CONFIG_IPQ_APSS_5018=y -+CONFIG_IPQ_GCC_5018=y -+# CONFIG_NET_SWITCHDEV is not set -+CONFIG_NUM_ALT_PARTITION=16 -+CONFIG_PINCTRL_IPQ5018=y -+# CONFIG_IPC_LOGGING is not set -+CONFIG_IPQ_SUBSYSTEM_DUMP=y -+# CONFIG_SPS is not set -+# CONFIG_SPS_SUPPORT_NDP_BAM is not set -+# CONFIG_CORESIGHT is not set -+# CONFIG_INPUT_PM8941_PWRKEY is not set -+CONFIG_MDIO_QCA=y -+# CONFIG_CRYPTO_ALL_CASES is not set -+# CONFIG_CRYPTO_DEV_QCOM_ICE is not set -+# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set -+# CONFIG_CRYPTO_SHA512 is not set -+# CONFIG_IIO is not set -+# CONFIG_IIO_BUFFER is not set -+# CONFIG_IIO_TRIGGER is not set -+CONFIG_PCIE_DW_PLAT=y -+CONFIG_PHY_IPQ_UNIPHY_PCIE=y -+CONFIG_VMSPLIT_2G=y -+# CONFIG_VMSPLIT_3G is not set -+# CONFIG_PPS is not set -+# CONFIG_PTP_1588_CLOCK is not set -+# CONFIG_DP83640_PHY is not set -+CONFIG_PWM_IPQ5018=y -+CONFIG_QCOM_APM=y -+# CONFIG_QCOM_DCC is not set -+# CONFIG_QCOM_SPMI_TEMP_ALARM is not set -+CONFIG_MMC_SDHCI_MSM_ICE=y -+CONFIG_USB_BAM=y -+CONFIG_MAILBOX=y -+# CONFIG_USB_QCOM_DIAG_BRIDGE is not set -+# CONFIG_USB_CONFIGFS_F_DIAG is not set -+# CONFIG_NF_IPV6_DUMMY_HEADER is not set -+# CONFIG_RMNET_DATA is not set -+# CONFIG_RMNET_DATA_DEBUG_PKT is not set -+CONFIG_MTD_NAND_SERIAL=y -+CONFIG_PAGE_SCOPE_MULTI_PAGE_READ=y -+# CONFIG_RMNET_DATA_FC is not set -+# CONFIG_CRYPTO_NO_ZERO_LEN_HASH is not set -+# CONFIG_CRYPTO_DISABLE_AES192_TEST is not set -+# CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE is not set -+# CONFIG_CPU_FREQ_GOV_PERFORMANCE is not set -+# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set -+# CONFIG_CPU_FREQ_GOV_ONDEMAND is not set -+# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set -+CONFIG_QTI_EUD=y -+CONFIG_USB_QCA_M31_PHY=y -+CONFIG_SQUASHFS_XZ=y -+# CONFIG_SQUASHFS_ZLIB is not set -+# CONFIG_JFFS2_LZMA is not set -+CONFIG_JFFS2_ZLIB=y -+# CONFIG_LZO_COMPRESS is not set -+# CONFIG_LZO_DECOMPRESS is not set -+CONFIG_XZ_DEC=y -+# CONFIG_XZ_DEC_X86 is not set -+# CONFIG_XZ_DEC_POWERPC is not set -+# CONFIG_XZ_DEC_IA64 is not set -+CONFIG_XZ_DEC_ARM=y -+# CONFIG_XZ_DEC_ARMTHUMB is not set -+# CONFIG_XZ_DEC_SPARC is not set -+CONFIG_XZ_DEC_BCJ=y -+# CONFIG_LZO_COMPRESS is not set -+# CONFIG_LZO_DECOMPRESS is not set -+# CONFIG_CRYPTO is not set -+CONFIG_QGIC2_MSI=y -+CONFIG_MTD_SPI_NOR_USE_4K_SECTORS=y -diff --git a/target/linux/ipq807x/ipq50xx/target.mk b/target/linux/ipq807x/ipq50xx/target.mk -new file mode 100644 -index 0000000000..649f398ba6 ---- /dev/null -+++ b/target/linux/ipq807x/ipq50xx/target.mk -@@ -0,0 +1,10 @@ -+ -+SUBTARGET:=ipq50xx -+BOARDNAME:=IPQ50XX -+CPU_TYPE:=cortex-a7 -+ -+DEFAULT_PACKAGES += ath11k-firmware-ipq50xx qca-nss-fw-ipq50xx -+ -+define Target/Description -+ Build firmware image for IPQ50xx SoC devices. -+endef -diff --git a/target/linux/ipq807x/ipq60xx/config-default b/target/linux/ipq807x/ipq60xx/config-default -new file mode 100644 -index 0000000000..3d7c59698e ---- /dev/null -+++ b/target/linux/ipq807x/ipq60xx/config-default -@@ -0,0 +1,122 @@ -+CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS=y -+CONFIG_ANDROID=y -+# CONFIG_ANDROID_BINDER_IPC is not set -+# CONFIG_AQ_PHY is not set -+CONFIG_ARCH_HIBERNATION_POSSIBLE=y -+CONFIG_ARCH_IPQ6018=y -+# CONFIG_ARCH_MSM8X60 is not set -+CONFIG_ARM_DMA_IOMMU_ALIGNMENT=8 -+CONFIG_ARM_DMA_USE_IOMMU=y -+CONFIG_ARM_HEAVY_MB=y -+CONFIG_ARM_QTI_IPQ60XX_CPUFREQ=y -+CONFIG_ARM_SMMU=y -+CONFIG_ASN1=y -+CONFIG_ASSOCIATIVE_ARRAY=y -+CONFIG_CACHE_L2X0=y -+CONFIG_CLZ_TAB=y -+CONFIG_CMA=y -+CONFIG_CMA_ALIGNMENT=8 -+CONFIG_CMA_AREAS=7 -+CONFIG_CMA_DEBUG=y -+CONFIG_CMA_DEBUGFS=y -+CONFIG_CMA_SIZE_MBYTES=0 -+CONFIG_CMA_SIZE_SEL_MBYTES=y -+CONFIG_CORESIGHT=y -+CONFIG_CORESIGHT_CSR=y -+CONFIG_CORESIGHT_CTI=y -+CONFIG_CORESIGHT_HWEVENT=y -+CONFIG_CORESIGHT_LINKS_AND_SINKS=y -+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y -+CONFIG_CORESIGHT_QCOM_REPLICATOR=y -+CONFIG_CORESIGHT_SINK_TPIU=y -+CONFIG_CORESIGHT_SOURCE_ETM3X=y -+CONFIG_CORESIGHT_SOURCE_ETM4X=y -+CONFIG_CORESIGHT_STM=y -+CONFIG_CORESIGHT_TPDA=y -+CONFIG_CORESIGHT_TPDM=y -+CONFIG_CRC_CCITT=m -+CONFIG_CRYPTO_AKCIPHER=y -+CONFIG_CRYPTO_AKCIPHER2=y -+CONFIG_CRYPTO_DRBG=y -+CONFIG_CRYPTO_DRBG_HMAC=y -+CONFIG_CRYPTO_DRBG_MENU=y -+CONFIG_CRYPTO_GF128MUL=y -+CONFIG_CRYPTO_GHASH=y -+CONFIG_CRYPTO_HASH=y -+CONFIG_CRYPTO_HMAC=y -+CONFIG_CRYPTO_JITTERENTROPY=y -+# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set -+CONFIG_CRYPTO_NULL2=y -+CONFIG_CRYPTO_PCOMP2=y -+CONFIG_CRYPTO_RNG_DEFAULT=y -+CONFIG_CRYPTO_SHA512=y -+CONFIG_DMA_CMA=y -+CONFIG_DMA_SHARED_BUFFER=y -+CONFIG_DT_IDLE_STATES=y -+CONFIG_EDAC_ATOMIC_SCRUB=y -+CONFIG_EDAC_SUPPORT=y -+CONFIG_EXTCON=y -+CONFIG_EXTCON_USB_GPIO=y -+CONFIG_FB_DEFERRED_IO=y -+CONFIG_FREEZER=y -+CONFIG_GPIO_WATCHDOG=y -+CONFIG_GPIO_WATCHDOG_ARCH_INITCALL=y -+CONFIG_IOMMU_API=y -+CONFIG_IOMMU_DMA=y -+CONFIG_IOMMU_IOVA=y -+CONFIG_IOMMU_IO_PGTABLE=y -+# CONFIG_IOMMU_IO_PGTABLE_ARMV7S is not set -+CONFIG_IOMMU_IO_PGTABLE_LPAE=y -+# CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST is not set -+CONFIG_IOMMU_SUPPORT=y -+CONFIG_ION=y -+CONFIG_ION_MSM=y -+CONFIG_IPQ_APSS_6018=y -+CONFIG_IPQ_GCC_6018=y -+CONFIG_IPQ_MEM_PROFILE=256 -+CONFIG_KASAN_SHADOW_OFFSET=0x5f000000 -+CONFIG_LEDS_PCA9956B=y -+CONFIG_MDIO_QCA=y -+CONFIG_MEMORY_ISOLATION=y -+CONFIG_MIGRATION=y -+CONFIG_MPILIB=y -+CONFIG_MSM_SECURE_BUFFER=y -+CONFIG_NEED_SG_DMA_LENGTH=y -+CONFIG_NET_SWITCHDEV=y -+CONFIG_NUM_ALT_PARTITION=16 -+CONFIG_OF_IOMMU=y -+CONFIG_OID_REGISTRY=y -+CONFIG_OUTER_CACHE=y -+CONFIG_OUTER_CACHE_SYNC=y -+CONFIG_PAGE_OFFSET=0x80000000 -+CONFIG_PINCTRL_IPQ6018=y -+# CONFIG_PKCS7_MESSAGE_PARSER is not set -+# CONFIG_PL310_ERRATA_588369 is not set -+# CONFIG_PL310_ERRATA_727915 is not set -+# CONFIG_PL310_ERRATA_753970 is not set -+# CONFIG_PL310_ERRATA_769419 is not set -+CONFIG_PPS=y -+CONFIG_PTP_1588_CLOCK=y -+CONFIG_PWM_IPQ4019=y -+CONFIG_QCOM_APM=y -+CONFIG_QCOM_DCC=y -+CONFIG_QCOM_QMI_HELPERS=y -+CONFIG_REGMAP_I2C=y -+CONFIG_REGMAP_SPMI=y -+CONFIG_REGULATOR_CPR3=y -+CONFIG_REGULATOR_CPR3_NPU=y -+CONFIG_REGULATOR_CPR4_APSS=y -+# CONFIG_SKB_FIXED_SIZE_2K is not set -+CONFIG_SOC_BUS=y -+CONFIG_SPS=y -+CONFIG_SPS_SUPPORT_NDP_BAM=y -+CONFIG_STAGING=y -+CONFIG_SUSPEND_FREEZER=y -+# CONFIG_USB_GADGET is not set -+CONFIG_USB_OHCI_LITTLE_ENDIAN=y -+# CONFIG_VFIO is not set -+CONFIG_VIRTIO=y -+CONFIG_VIRTUALIZATION=y -+CONFIG_VMSPLIT_2G=y -+# CONFIG_VMSPLIT_3G is not set -+CONFIG_WANT_DEV_COREDUMP=y -diff --git a/target/linux/ipq807x/ipq60xx/profiles/default.mk b/target/linux/ipq807x/ipq60xx/profiles/default.mk -new file mode 100644 -index 0000000000..f47e73acfc ---- /dev/null -+++ b/target/linux/ipq807x/ipq60xx/profiles/default.mk -@@ -0,0 +1,9 @@ -+define Profile/Default -+ NAME:=Default Profile (minimum package set) -+endef -+ -+define Profile/Default/Description -+Default package set compatible with most boards. -+endef -+$(eval $(call Profile,Default)) -+ -diff --git a/target/linux/ipq807x/ipq60xx/target.mk b/target/linux/ipq807x/ipq60xx/target.mk -new file mode 100644 -index 0000000000..cf2bf7b9cd ---- /dev/null -+++ b/target/linux/ipq807x/ipq60xx/target.mk -@@ -0,0 +1,8 @@ -+SUBTARGET:=ipq60xx -+BOARDNAME:=IPQ60xx based boards -+ -+DEFAULT_PACKAGES += ath11k-firmware-ipq60xx qca-nss-fw-ipq60xx -+ -+define Target/Description -+ Build images for IPQ60xx systems. -+endef -diff --git a/target/linux/ipq807x/ipq807x/config-default b/target/linux/ipq807x/ipq807x/config-default -new file mode 100644 -index 0000000000..f1e8aadc9c ---- /dev/null -+++ b/target/linux/ipq807x/ipq807x/config-default -@@ -0,0 +1,78 @@ -+# CONFIG_AHCI_IPQ is not set -+CONFIG_ARCH_IPQ807x=y -+# CONFIG_DIAGFWD_BRIDGE_CODE is not set -+CONFIG_IPQ_ADSS_807x=y -+CONFIG_IPQ_APSS_807x=y -+CONFIG_IPQ_GCC_807x=y -+CONFIG_NET_SWITCHDEV=y -+CONFIG_NUM_ALT_PARTITION=16 -+CONFIG_PINCTRL_IPQ807x=y -+# CONFIG_IPC_LOGGING is not set -+CONFIG_IPQ_SUBSYSTEM_DUMP=y -+CONFIG_SPS=y -+CONFIG_SPS_SUPPORT_NDP_BAM=y -+CONFIG_CORESIGHT=y -+CONFIG_CORESIGHT_CSR=y -+CONFIG_CORESIGHT_CTI=y -+CONFIG_CORESIGHT_EVENT=y -+CONFIG_CORESIGHT_HWEVENT=y -+CONFIG_CORESIGHT_LINKS_AND_SINKS=y -+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y -+CONFIG_CORESIGHT_QCOM_REPLICATOR=y -+CONFIG_CORESIGHT_STREAM=m -+# CONFIG_INPUT_PM8941_PWRKEY is not set -+CONFIG_MDIO_QCA=y -+# CONFIG_CRYPTO_ALL_CASES is not set -+CONFIG_CRYPTO_DEV_QCOM_ICE=y -+# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set -+CONFIG_CRYPTO_SHA512=y -+# CONFIG_CORESIGHT_QPDI is not set -+# CONFIG_CORESIGHT_SINK_ETBV10 is not set -+CONFIG_CORESIGHT_SINK_TPIU=y -+# CONFIG_CORESIGHT_SOURCE_DUMMY is not set -+CONFIG_CORESIGHT_SOURCE_ETM3X=y -+CONFIG_CORESIGHT_SOURCE_ETM4X=y -+# CONFIG_CORESIGHT_REMOTE_ETM is not set -+CONFIG_CORESIGHT_STM=y -+CONFIG_CORESIGHT_TPDA=y -+CONFIG_CORESIGHT_TPDM=y -+CONFIG_AQUANTIA_PHY=y -+# CONFIG_CORESIGHT_TPDM_DEFAULT_ENABLE is not set -+CONFIG_IIO=y -+# CONFIG_IIO_BUFFER is not set -+# CONFIG_IIO_TRIGGER is not set -+CONFIG_PCIE_DW_PLAT=y -+CONFIG_VMSPLIT_2G=y -+# CONFIG_VMSPLIT_3G is not set -+CONFIG_PPS=y -+CONFIG_PTP_1588_CLOCK=y -+# CONFIG_DP83640_PHY is not set -+CONFIG_PWM_IPQ4019=y -+CONFIG_QCOM_APM=y -+CONFIG_QCOM_DCC=y -+# CONFIG_QCOM_SPMI_TEMP_ALARM is not set -+CONFIG_QCOM_SPMI_VADC=y -+CONFIG_REGMAP_ALLOW_WRITE_DEBUGFS=y -+CONFIG_REGULATOR_CPR3=y -+CONFIG_REGULATOR_CPR3_NPU=y -+CONFIG_REGULATOR_CPR4_APSS=y -+CONFIG_MMC_SDHCI_MSM_ICE=y -+CONFIG_USB_BAM=y -+CONFIG_USB_QCOM_KS_BRIDGE=m -+CONFIG_MAILBOX=y -+# CONFIG_USB_QCOM_DIAG_BRIDGE is not set -+# CONFIG_USB_CONFIGFS_F_DIAG is not set -+# CONFIG_NF_IPV6_DUMMY_HEADER is not set -+CONFIG_RMNET=y -+CONFIG_RMNET_DATA=y -+CONFIG_RMNET_DATA_DEBUG_PKT=y -+# CONFIG_RMNET_DATA_FC is not set -+CONFIG_QCOM_QMI_RMNET=y -+CONFIG_QCOM_QMI_DFC=y -+CONFIG_QCOM_QMI_POWER_COLLAPSE=y -+CONFIG_RMNET_CTL=y -+CONFIG_RMNET_CTL_DEBUG=y -+CONFIG_MHI_BUS_TEST=y -+CONFIG_MHI_DEBUG=y -+CONFIG_MHI_NETDEV=y -+CONFIG_MHI_UCI=y -diff --git a/target/linux/ipq807x/ipq807x/profiles/default.mk b/target/linux/ipq807x/ipq807x/profiles/default.mk -new file mode 100644 -index 0000000000..f47e73acfc ---- /dev/null -+++ b/target/linux/ipq807x/ipq807x/profiles/default.mk -@@ -0,0 +1,9 @@ -+define Profile/Default -+ NAME:=Default Profile (minimum package set) -+endef -+ -+define Profile/Default/Description -+Default package set compatible with most boards. -+endef -+$(eval $(call Profile,Default)) -+ -diff --git a/target/linux/ipq807x/ipq807x/target.mk b/target/linux/ipq807x/ipq807x/target.mk -new file mode 100644 -index 0000000000..7c24b66e60 ---- /dev/null -+++ b/target/linux/ipq807x/ipq807x/target.mk -@@ -0,0 +1,7 @@ -+SUBTARGET:=ipq807x -+BOARDNAME:=IPQ807x based boards -+ -+DEFAULT_PACKAGES += ath11k-firmware-ipq807x qca-nss-fw-ipq807x -+define Target/Description -+ Build images for IPQ807x systems. -+endef -diff --git a/target/linux/ipq807x/modules.mk b/target/linux/ipq807x/modules.mk -new file mode 100644 -index 0000000000..0223ff2590 ---- /dev/null -+++ b/target/linux/ipq807x/modules.mk -@@ -0,0 +1,61 @@ -+define KernelPackage/usb-phy-ipq807x -+ TITLE:=DWC3 USB QCOM PHY driver for IPQ807x -+ DEPENDS:=@TARGET_ipq807x -+ KCONFIG:= \ -+ CONFIG_USB_QCOM_QUSB_PHY \ -+ CONFIG_USB_QCOM_QMP_PHY -+ FILES:= \ -+ $(LINUX_DIR)/drivers/usb/phy/phy-msm-qusb.ko \ -+ $(LINUX_DIR)/drivers/usb/phy/phy-msm-ssusb-qmp.ko -+ AUTOLOAD:=$(call AutoLoad,45,phy-msm-qusb phy-msm-ssusb-qmp,1) -+ $(call AddDepends/usb) -+endef -+ -+define KernelPackage/usb-phy-ipq807x/description -+ This driver provides support for the USB PHY drivers -+ within the IPQ807x SoCs. -+endef -+ -+$(eval $(call KernelPackage,usb-phy-ipq807x)) -+ -+ -+define KernelPackage/qrtr_mproc -+ TITLE:= Ath11k Specific kernel configs for IPQ807x and IPQ60xx -+ DEPENDS+= @TARGET_ipq807x -+ KCONFIG:= \ -+ CONFIG_QRTR=y \ -+ CONFIG_QRTR_MHI=y \ -+ CONFIG_MHI_BUS=y \ -+ CONFIG_MHI_QTI=y \ -+ CONFIG_QCOM_APCS_IPC=y \ -+ CONFIG_QCOM_GLINK_SSR=y \ -+ CONFIG_QCOM_Q6V5_WCSS=y \ -+ CONFIG_MSM_RPM_RPMSG=y \ -+ CONFIG_RPMSG_QCOM_GLINK_RPM=y \ -+ CONFIG_REGULATOR_RPM_GLINK=y \ -+ CONFIG_QCOM_SYSMON=y \ -+ CONFIG_RPMSG=y \ -+ CONFIG_RPMSG_CHAR=y \ -+ CONFIG_RPMSG_QCOM_GLINK_SMEM=y \ -+ CONFIG_RPMSG_QCOM_SMD=y \ -+ CONFIG_QRTR_SMD=y \ -+ CONFIG_QCOM_QMI_HELPERS=y \ -+ CONFIG_SAMPLES=y \ -+ CONFIG_SAMPLE_QMI_CLIENT=m \ -+ CONFIG_SAMPLE_TRACE_EVENTS=n \ -+ CONFIG_SAMPLE_KOBJECT=n \ -+ CONFIG_SAMPLE_KPROBES=n \ -+ CONFIG_SAMPLE_KRETPROBES=n \ -+ CONFIG_SAMPLE_HW_BREAKPOINT=n \ -+ CONFIG_SAMPLE_KFIFO=n \ -+ CONFIG_SAMPLE_CONFIGFS=n \ -+ CONFIG_SAMPLE_RPMSG_CLIENT=n \ -+ CONFIG_MAILBOX=y \ -+ CONFIG_DIAG_OVER_QRTR=y -+endef -+ -+define KernelPackage/qrtr_mproc/description -+Kernel configs for ath11k support specific to ipq807x and IPQ60xx -+endef -+ -+$(eval $(call KernelPackage,qrtr_mproc)) -diff --git a/target/linux/ipq807x/patches/100-qrtr-ns.patch b/target/linux/ipq807x/patches/100-qrtr-ns.patch -new file mode 100644 -index 0000000000..850e644778 ---- /dev/null -+++ b/target/linux/ipq807x/patches/100-qrtr-ns.patch -@@ -0,0 +1,976 @@ -+Index: linux-4.4.60/net/qrtr/ns.c -+=================================================================== -+--- /dev/null -++++ linux-4.4.60/net/qrtr/ns.c -+@@ -0,0 +1,760 @@ -++// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -++/* -++ * Copyright (c) 2015, Sony Mobile Communications Inc. -++ * Copyright (c) 2013, The Linux Foundation. All rights reserved. -++ * Copyright (c) 2020, Linaro Ltd. -++ */ -++ -++#include -++#include -++#include -++#include -++ -++#include "qrtr.h" -++ -++#define CREATE_TRACE_POINTS -++#include -++ -++static RADIX_TREE(nodes, GFP_KERNEL); -++ -++static struct { -++ struct socket *sock; -++ struct sockaddr_qrtr bcast_sq; -++ struct list_head lookups; -++ struct workqueue_struct *workqueue; -++ struct work_struct work; -++ int local_node; -++} qrtr_ns; -++ -++static const char * const qrtr_ctrl_pkt_strings[] = { -++ [QRTR_TYPE_HELLO] = "hello", -++ [QRTR_TYPE_BYE] = "bye", -++ [QRTR_TYPE_NEW_SERVER] = "new-server", -++ [QRTR_TYPE_DEL_SERVER] = "del-server", -++ [QRTR_TYPE_DEL_CLIENT] = "del-client", -++ [QRTR_TYPE_RESUME_TX] = "resume-tx", -++ [QRTR_TYPE_EXIT] = "exit", -++ [QRTR_TYPE_PING] = "ping", -++ [QRTR_TYPE_NEW_LOOKUP] = "new-lookup", -++ [QRTR_TYPE_DEL_LOOKUP] = "del-lookup", -++}; -++ -++struct qrtr_server_filter { -++ unsigned int service; -++ unsigned int instance; -++ unsigned int ifilter; -++}; -++ -++struct qrtr_lookup { -++ unsigned int service; -++ unsigned int instance; -++ -++ struct sockaddr_qrtr sq; -++ struct list_head li; -++}; -++ -++struct qrtr_server { -++ unsigned int service; -++ unsigned int instance; -++ -++ unsigned int node; -++ unsigned int port; -++ -++ struct list_head qli; -++}; -++ -++struct qrtr_node { -++ unsigned int id; -++ struct radix_tree_root servers; -++}; -++ -++static struct qrtr_node *node_get(unsigned int node_id) -++{ -++ struct qrtr_node *node; -++ -++ node = radix_tree_lookup(&nodes, node_id); -++ if (node) -++ return node; -++ -++ /* If node didn't exist, allocate and insert it to the tree */ -++ node = kzalloc(sizeof(*node), GFP_KERNEL); -++ if (!node) -++ return NULL; -++ -++ node->id = node_id; -++ -++ radix_tree_insert(&nodes, node_id, node); -++ -++ return node; -++} -++ -++static int server_match(const struct qrtr_server *srv, -++ const struct qrtr_server_filter *f) -++{ -++ unsigned int ifilter = f->ifilter; -++ -++ if (f->service != 0 && srv->service != f->service) -++ return 0; -++ if (!ifilter && f->instance) -++ ifilter = ~0; -++ -++ return (srv->instance & ifilter) == f->instance; -++} -++ -++static int service_announce_new(struct sockaddr_qrtr *dest, -++ struct qrtr_server *srv) -++{ -++ struct qrtr_ctrl_pkt pkt; -++ struct msghdr msg = { }; -++ struct kvec iv; -++ -++ trace_qrtr_ns_service_announce_new(srv->service, srv->instance, -++ srv->node, srv->port); -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = cpu_to_le32(QRTR_TYPE_NEW_SERVER); -++ pkt.server.service = cpu_to_le32(srv->service); -++ pkt.server.instance = cpu_to_le32(srv->instance); -++ pkt.server.node = cpu_to_le32(srv->node); -++ pkt.server.port = cpu_to_le32(srv->port); -++ -++ msg.msg_name = (struct sockaddr *)dest; -++ msg.msg_namelen = sizeof(*dest); -++ -++ return kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++} -++ -++static int service_announce_del(struct sockaddr_qrtr *dest, -++ struct qrtr_server *srv) -++{ -++ struct qrtr_ctrl_pkt pkt; -++ struct msghdr msg = { }; -++ struct kvec iv; -++ int ret; -++ -++ trace_qrtr_ns_service_announce_del(srv->service, srv->instance, -++ srv->node, srv->port); -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = cpu_to_le32(QRTR_TYPE_DEL_SERVER); -++ pkt.server.service = cpu_to_le32(srv->service); -++ pkt.server.instance = cpu_to_le32(srv->instance); -++ pkt.server.node = cpu_to_le32(srv->node); -++ pkt.server.port = cpu_to_le32(srv->port); -++ -++ msg.msg_name = (struct sockaddr *)dest; -++ msg.msg_namelen = sizeof(*dest); -++ -++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++ if (ret < 0) -++ pr_err("failed to announce del service\n"); -++ -++ return ret; -++} -++ -++static void lookup_notify(struct sockaddr_qrtr *to, struct qrtr_server *srv, -++ bool new) -++{ -++ struct qrtr_ctrl_pkt pkt; -++ struct msghdr msg = { }; -++ struct kvec iv; -++ int ret; -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = new ? cpu_to_le32(QRTR_TYPE_NEW_SERVER) : -++ cpu_to_le32(QRTR_TYPE_DEL_SERVER); -++ if (srv) { -++ pkt.server.service = cpu_to_le32(srv->service); -++ pkt.server.instance = cpu_to_le32(srv->instance); -++ pkt.server.node = cpu_to_le32(srv->node); -++ pkt.server.port = cpu_to_le32(srv->port); -++ } -++ -++ msg.msg_name = (struct sockaddr *)to; -++ msg.msg_namelen = sizeof(*to); -++ -++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++ if (ret < 0) -++ pr_err("failed to send lookup notification\n"); -++} -++ -++static int announce_servers(struct sockaddr_qrtr *sq) -++{ -++ struct radix_tree_iter iter; -++ struct qrtr_server *srv; -++ struct qrtr_node *node; -++ void __rcu **slot; -++ int ret; -++ -++ node = node_get(qrtr_ns.local_node); -++ if (!node) -++ return 0; -++ -++ /* Announce the list of servers registered in this node */ -++ radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { -++ srv = radix_tree_deref_slot(slot); -++ -++ ret = service_announce_new(sq, srv); -++ if (ret < 0) { -++ pr_err("failed to announce new service\n"); -++ return ret; -++ } -++ } -++ -++ return 0; -++} -++ -++static struct qrtr_server *server_add(unsigned int service, -++ unsigned int instance, -++ unsigned int node_id, -++ unsigned int port) -++{ -++ struct qrtr_server *srv; -++ struct qrtr_server *old; -++ struct qrtr_node *node; -++ -++ if (!service || !port) -++ return NULL; -++ -++ srv = kzalloc(sizeof(*srv), GFP_KERNEL); -++ if (!srv) -++ return NULL; -++ -++ srv->service = service; -++ srv->instance = instance; -++ srv->node = node_id; -++ srv->port = port; -++ -++ node = node_get(node_id); -++ if (!node) -++ goto err; -++ -++ /* Delete the old server on the same port */ -++ old = radix_tree_lookup(&node->servers, port); -++ if (old) { -++ radix_tree_delete(&node->servers, port); -++ kfree(old); -++ } -++ -++ radix_tree_insert(&node->servers, port, srv); -++ -++ trace_qrtr_ns_server_add(srv->service, srv->instance, -++ srv->node, srv->port); -++ -++ return srv; -++ -++err: -++ kfree(srv); -++ return NULL; -++} -++ -++static int server_del(struct qrtr_node *node, unsigned int port) -++{ -++ struct qrtr_lookup *lookup; -++ struct qrtr_server *srv; -++ struct list_head *li; -++ -++ srv = radix_tree_lookup(&node->servers, port); -++ if (!srv) -++ return -ENOENT; -++ -++ radix_tree_delete(&node->servers, port); -++ -++ /* Broadcast the removal of local servers */ -++ if (srv->node == qrtr_ns.local_node) -++ service_announce_del(&qrtr_ns.bcast_sq, srv); -++ -++ /* Announce the service's disappearance to observers */ -++ list_for_each(li, &qrtr_ns.lookups) { -++ lookup = container_of(li, struct qrtr_lookup, li); -++ if (lookup->service && lookup->service != srv->service) -++ continue; -++ if (lookup->instance && lookup->instance != srv->instance) -++ continue; -++ -++ lookup_notify(&lookup->sq, srv, false); -++ } -++ -++ kfree(srv); -++ -++ return 0; -++} -++ -++static int say_hello(struct sockaddr_qrtr *dest) -++{ -++ struct qrtr_ctrl_pkt pkt; -++ struct msghdr msg = { }; -++ struct kvec iv; -++ int ret; -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = cpu_to_le32(QRTR_TYPE_HELLO); -++ -++ msg.msg_name = (struct sockaddr *)dest; -++ msg.msg_namelen = sizeof(*dest); -++ -++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++ if (ret < 0) -++ pr_err("failed to send hello msg\n"); -++ -++ return ret; -++} -++ -++/* Announce the list of servers registered on the local node */ -++static int ctrl_cmd_hello(struct sockaddr_qrtr *sq) -++{ -++ int ret; -++ -++ ret = say_hello(sq); -++ if (ret < 0) -++ return ret; -++ -++ return announce_servers(sq); -++} -++ -++static int ctrl_cmd_bye(struct sockaddr_qrtr *from) -++{ -++ struct qrtr_node *local_node; -++ struct radix_tree_iter iter; -++ struct qrtr_ctrl_pkt pkt; -++ struct qrtr_server *srv; -++ struct sockaddr_qrtr sq; -++ struct msghdr msg = { }; -++ struct qrtr_node *node; -++ void __rcu **slot; -++ struct kvec iv; -++ int ret; -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ node = node_get(from->sq_node); -++ if (!node) -++ return 0; -++ -++ /* Advertise removal of this client to all servers of remote node */ -++ radix_tree_for_each_slot(slot, &node->servers, &iter, 0) { -++ srv = radix_tree_deref_slot(slot); -++ server_del(node, srv->port); -++ } -++ -++ /* Advertise the removal of this client to all local servers */ -++ local_node = node_get(qrtr_ns.local_node); -++ if (!local_node) -++ return 0; -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = cpu_to_le32(QRTR_TYPE_BYE); -++ pkt.client.node = cpu_to_le32(from->sq_node); -++ -++ radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { -++ srv = radix_tree_deref_slot(slot); -++ -++ sq.sq_family = AF_QIPCRTR; -++ sq.sq_node = srv->node; -++ sq.sq_port = srv->port; -++ -++ msg.msg_name = (struct sockaddr *)&sq; -++ msg.msg_namelen = sizeof(sq); -++ -++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++ if (ret < 0) { -++ pr_err("failed to send bye cmd\n"); -++ return ret; -++ } -++ } -++ -++ return 0; -++} -++ -++static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, -++ unsigned int node_id, unsigned int port) -++{ -++ struct qrtr_node *local_node; -++ struct radix_tree_iter iter; -++ struct qrtr_lookup *lookup; -++ struct qrtr_ctrl_pkt pkt; -++ struct msghdr msg = { }; -++ struct qrtr_server *srv; -++ struct sockaddr_qrtr sq; -++ struct qrtr_node *node; -++ struct list_head *tmp; -++ struct list_head *li; -++ void __rcu **slot; -++ struct kvec iv; -++ int ret; -++ -++ iv.iov_base = &pkt; -++ iv.iov_len = sizeof(pkt); -++ -++ /* Don't accept spoofed messages */ -++ if (from->sq_node != node_id) -++ return -EINVAL; -++ -++ /* Local DEL_CLIENT messages comes from the port being closed */ -++ if (from->sq_node == qrtr_ns.local_node && from->sq_port != port) -++ return -EINVAL; -++ -++ /* Remove any lookups by this client */ -++ list_for_each_safe(li, tmp, &qrtr_ns.lookups) { -++ lookup = container_of(li, struct qrtr_lookup, li); -++ if (lookup->sq.sq_node != node_id) -++ continue; -++ if (lookup->sq.sq_port != port) -++ continue; -++ -++ list_del(&lookup->li); -++ kfree(lookup); -++ } -++ -++ /* Remove the server belonging to this port */ -++ node = node_get(node_id); -++ if (node) -++ server_del(node, port); -++ -++ /* Advertise the removal of this client to all local servers */ -++ local_node = node_get(qrtr_ns.local_node); -++ if (!local_node) -++ return 0; -++ -++ memset(&pkt, 0, sizeof(pkt)); -++ pkt.cmd = cpu_to_le32(QRTR_TYPE_DEL_CLIENT); -++ pkt.client.node = cpu_to_le32(node_id); -++ pkt.client.port = cpu_to_le32(port); -++ -++ radix_tree_for_each_slot(slot, &local_node->servers, &iter, 0) { -++ srv = radix_tree_deref_slot(slot); -++ -++ sq.sq_family = AF_QIPCRTR; -++ sq.sq_node = srv->node; -++ sq.sq_port = srv->port; -++ -++ msg.msg_name = (struct sockaddr *)&sq; -++ msg.msg_namelen = sizeof(sq); -++ -++ ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); -++ if (ret < 0) { -++ pr_err("failed to send del client cmd\n"); -++ return ret; -++ } -++ } -++ -++ return 0; -++} -++ -++static int ctrl_cmd_new_server(struct sockaddr_qrtr *from, -++ unsigned int service, unsigned int instance, -++ unsigned int node_id, unsigned int port) -++{ -++ struct qrtr_lookup *lookup; -++ struct qrtr_server *srv; -++ struct list_head *li; -++ int ret = 0; -++ -++ /* Ignore specified node and port for local servers */ -++ if (from->sq_node == qrtr_ns.local_node) { -++ node_id = from->sq_node; -++ port = from->sq_port; -++ } -++ -++ /* Don't accept spoofed messages */ -++ if (from->sq_node != node_id) -++ return -EINVAL; -++ -++ srv = server_add(service, instance, node_id, port); -++ if (!srv) -++ return -EINVAL; -++ -++ if (srv->node == qrtr_ns.local_node) { -++ ret = service_announce_new(&qrtr_ns.bcast_sq, srv); -++ if (ret < 0) { -++ pr_err("failed to announce new service\n"); -++ return ret; -++ } -++ } -++ -++ /* Notify any potential lookups about the new server */ -++ list_for_each(li, &qrtr_ns.lookups) { -++ lookup = container_of(li, struct qrtr_lookup, li); -++ if (lookup->service && lookup->service != service) -++ continue; -++ if (lookup->instance && lookup->instance != instance) -++ continue; -++ -++ lookup_notify(&lookup->sq, srv, true); -++ } -++ -++ return ret; -++} -++ -++static int ctrl_cmd_del_server(struct sockaddr_qrtr *from, -++ unsigned int service, unsigned int instance, -++ unsigned int node_id, unsigned int port) -++{ -++ struct qrtr_node *node; -++ -++ /* Ignore specified node and port for local servers*/ -++ if (from->sq_node == qrtr_ns.local_node) { -++ node_id = from->sq_node; -++ port = from->sq_port; -++ } -++ -++ /* Don't accept spoofed messages */ -++ if (from->sq_node != node_id) -++ return -EINVAL; -++ -++ /* Local servers may only unregister themselves */ -++ if (from->sq_node == qrtr_ns.local_node && from->sq_port != port) -++ return -EINVAL; -++ -++ node = node_get(node_id); -++ if (!node) -++ return -ENOENT; -++ -++ return server_del(node, port); -++} -++ -++static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, -++ unsigned int service, unsigned int instance) -++{ -++ struct radix_tree_iter node_iter; -++ struct qrtr_server_filter filter; -++ struct radix_tree_iter srv_iter; -++ struct qrtr_lookup *lookup; -++ struct qrtr_node *node; -++ void __rcu **node_slot; -++ void __rcu **srv_slot; -++ -++ /* Accept only local observers */ -++ if (from->sq_node != qrtr_ns.local_node) -++ return -EINVAL; -++ -++ lookup = kzalloc(sizeof(*lookup), GFP_KERNEL); -++ if (!lookup) -++ return -ENOMEM; -++ -++ lookup->sq = *from; -++ lookup->service = service; -++ lookup->instance = instance; -++ list_add_tail(&lookup->li, &qrtr_ns.lookups); -++ -++ memset(&filter, 0, sizeof(filter)); -++ filter.service = service; -++ filter.instance = instance; -++ -++ radix_tree_for_each_slot(node_slot, &nodes, &node_iter, 0) { -++ node = radix_tree_deref_slot(node_slot); -++ -++ radix_tree_for_each_slot(srv_slot, &node->servers, -++ &srv_iter, 0) { -++ struct qrtr_server *srv; -++ -++ srv = radix_tree_deref_slot(srv_slot); -++ if (!server_match(srv, &filter)) -++ continue; -++ -++ lookup_notify(from, srv, true); -++ } -++ } -++ -++ /* Empty notification, to indicate end of listing */ -++ lookup_notify(from, NULL, true); -++ -++ return 0; -++} -++ -++static void ctrl_cmd_del_lookup(struct sockaddr_qrtr *from, -++ unsigned int service, unsigned int instance) -++{ -++ struct qrtr_lookup *lookup; -++ struct list_head *tmp; -++ struct list_head *li; -++ -++ list_for_each_safe(li, tmp, &qrtr_ns.lookups) { -++ lookup = container_of(li, struct qrtr_lookup, li); -++ if (lookup->sq.sq_node != from->sq_node) -++ continue; -++ if (lookup->sq.sq_port != from->sq_port) -++ continue; -++ if (lookup->service != service) -++ continue; -++ if (lookup->instance && lookup->instance != instance) -++ continue; -++ -++ list_del(&lookup->li); -++ kfree(lookup); -++ } -++} -++ -++static void qrtr_ns_worker(struct work_struct *work) -++{ -++ const struct qrtr_ctrl_pkt *pkt; -++ size_t recv_buf_size = 4096; -++ struct sockaddr_qrtr sq; -++ struct msghdr msg = { }; -++ unsigned int cmd; -++ ssize_t msglen; -++ void *recv_buf; -++ struct kvec iv; -++ int ret; -++ -++ msg.msg_name = (struct sockaddr *)&sq; -++ msg.msg_namelen = sizeof(sq); -++ -++ recv_buf = kzalloc(recv_buf_size, GFP_KERNEL); -++ if (!recv_buf) -++ return; -++ -++ for (;;) { -++ iv.iov_base = recv_buf; -++ iv.iov_len = recv_buf_size; -++ -++ msglen = kernel_recvmsg(qrtr_ns.sock, &msg, &iv, 1, -++ iv.iov_len, MSG_DONTWAIT); -++ -++ if (msglen == -EAGAIN) -++ break; -++ -++ if (msglen < 0) { -++ pr_err("error receiving packet: %zd\n", msglen); -++ break; -++ } -++ -++ pkt = recv_buf; -++ cmd = le32_to_cpu(pkt->cmd); -++ if (cmd < ARRAY_SIZE(qrtr_ctrl_pkt_strings) && -++ qrtr_ctrl_pkt_strings[cmd]) -++ trace_qrtr_ns_message(qrtr_ctrl_pkt_strings[cmd], -++ sq.sq_node, sq.sq_port); -++ -++ ret = 0; -++ switch (cmd) { -++ case QRTR_TYPE_HELLO: -++ ret = ctrl_cmd_hello(&sq); -++ break; -++ case QRTR_TYPE_BYE: -++ ret = ctrl_cmd_bye(&sq); -++ break; -++ case QRTR_TYPE_DEL_CLIENT: -++ ret = ctrl_cmd_del_client(&sq, -++ le32_to_cpu(pkt->client.node), -++ le32_to_cpu(pkt->client.port)); -++ break; -++ case QRTR_TYPE_NEW_SERVER: -++ ret = ctrl_cmd_new_server(&sq, -++ le32_to_cpu(pkt->server.service), -++ le32_to_cpu(pkt->server.instance), -++ le32_to_cpu(pkt->server.node), -++ le32_to_cpu(pkt->server.port)); -++ break; -++ case QRTR_TYPE_DEL_SERVER: -++ ret = ctrl_cmd_del_server(&sq, -++ le32_to_cpu(pkt->server.service), -++ le32_to_cpu(pkt->server.instance), -++ le32_to_cpu(pkt->server.node), -++ le32_to_cpu(pkt->server.port)); -++ break; -++ case QRTR_TYPE_EXIT: -++ case QRTR_TYPE_PING: -++ case QRTR_TYPE_RESUME_TX: -++ break; -++ case QRTR_TYPE_NEW_LOOKUP: -++ ret = ctrl_cmd_new_lookup(&sq, -++ le32_to_cpu(pkt->server.service), -++ le32_to_cpu(pkt->server.instance)); -++ break; -++ case QRTR_TYPE_DEL_LOOKUP: -++ ctrl_cmd_del_lookup(&sq, -++ le32_to_cpu(pkt->server.service), -++ le32_to_cpu(pkt->server.instance)); -++ break; -++ } -++ -++ if (ret < 0) -++ pr_err("failed while handling packet from %d:%d", -++ sq.sq_node, sq.sq_port); -++ } -++ -++ kfree(recv_buf); -++} -++ -++static void qrtr_ns_data_ready(struct sock *sk) -++{ -++ queue_work(qrtr_ns.workqueue, &qrtr_ns.work); -++} -++ -++void qrtr_ns_init(void) -++{ -++ struct sockaddr_qrtr sq; -++ int sl = sizeof(sq); -++ int ret; -++ -++ INIT_LIST_HEAD(&qrtr_ns.lookups); -++ INIT_WORK(&qrtr_ns.work, qrtr_ns_worker); -++ -++ ret = sock_create_kern(&init_net, AF_QIPCRTR, SOCK_DGRAM, -++ PF_QIPCRTR, &qrtr_ns.sock); -++ if (ret < 0) -++ return; -++ -++ ret = kernel_getsockname(qrtr_ns.sock, (struct sockaddr *)&sq, &sl); -++ if (ret < 0) { -++ pr_err("failed to get socket name\n"); -++ goto err_sock; -++ } -++ -++ qrtr_ns.workqueue = alloc_workqueue("qrtr_ns_handler", WQ_UNBOUND, 1); -++ if (!qrtr_ns.workqueue) -++ goto err_sock; -++ -++ qrtr_ns.sock->sk->sk_data_ready = qrtr_ns_data_ready; -++ -++ sq.sq_port = QRTR_PORT_CTRL; -++ qrtr_ns.local_node = sq.sq_node; -++ -++ ret = kernel_bind(qrtr_ns.sock, (struct sockaddr *)&sq, sizeof(sq)); -++ if (ret < 0) { -++ pr_err("failed to bind to socket\n"); -++ goto err_wq; -++ } -++ -++ qrtr_ns.bcast_sq.sq_family = AF_QIPCRTR; -++ qrtr_ns.bcast_sq.sq_node = QRTR_NODE_BCAST; -++ qrtr_ns.bcast_sq.sq_port = QRTR_PORT_CTRL; -++ -++ ret = say_hello(&qrtr_ns.bcast_sq); -++ if (ret < 0) -++ goto err_wq; -++ -++ return; -++ -++err_wq: -++ destroy_workqueue(qrtr_ns.workqueue); -++err_sock: -++ sock_release(qrtr_ns.sock); -++} -++EXPORT_SYMBOL_GPL(qrtr_ns_init); -++ -++void qrtr_ns_remove(void) -++{ -++ cancel_work_sync(&qrtr_ns.work); -++ destroy_workqueue(qrtr_ns.workqueue); -++ sock_release(qrtr_ns.sock); -++} -++EXPORT_SYMBOL_GPL(qrtr_ns_remove); -++ -++MODULE_AUTHOR("Manivannan Sadhasivam "); -++MODULE_DESCRIPTION("Qualcomm IPC Router Nameservice"); -++MODULE_LICENSE("Dual BSD/GPL"); -+Index: linux-4.4.60/net/qrtr/qrtr.c -+=================================================================== -+--- linux-4.4.60.orig/net/qrtr/qrtr.c -++++ linux-4.4.60/net/qrtr/qrtr.c -+@@ -135,6 +135,8 @@ static DEFINE_IDR(qrtr_ports); -+ static DEFINE_MUTEX(qrtr_port_lock); -+ static DEFINE_MUTEX(qrtr_node_locking); -+ -++static struct delayed_work qrtr_ns_work; -++ -+ /** -+ * struct qrtr_node - endpoint node -+ * @ep_lock: lock for endpoint management and callbacks -+@@ -1765,33 +1767,6 @@ static int qrtr_create(struct net *net, -+ return 0; -+ } -+ -+-static const struct nla_policy qrtr_policy[IFA_MAX + 1] = { -+- [IFA_LOCAL] = { .type = NLA_U32 }, -+-}; -+- -+-static int qrtr_addr_doit(struct sk_buff *skb, struct nlmsghdr *nlh) -+-{ -+- struct nlattr *tb[IFA_MAX + 1]; -+- struct ifaddrmsg *ifm; -+- int rc; -+- -+- if (!netlink_capable(skb, CAP_NET_ADMIN)) -+- return -EPERM; -+- -+- ASSERT_RTNL(); -+- -+- rc = nlmsg_parse(nlh, sizeof(*ifm), tb, IFA_MAX, qrtr_policy); -+- if (rc < 0) -+- return rc; -+- -+- ifm = nlmsg_data(nlh); -+- if (!tb[IFA_LOCAL]) -+- return -EINVAL; -+- -+- qrtr_local_nid = nla_get_u32(tb[IFA_LOCAL]); -+- return 0; -+-} -+- -+ static const struct net_proto_family qrtr_family = { -+ .owner = THIS_MODULE, -+ .family = AF_QIPCRTR, -+@@ -1811,7 +1786,8 @@ static int __init qrtr_proto_init(void) -+ proto_unregister(&qrtr_proto); -+ return rc; -+ } -+- rtnl_register(PF_QIPCRTR, RTM_NEWADDR, qrtr_addr_doit, NULL, NULL); -++ -++ qrtr_ns_init(); -+ -+ return 0; -+ } -+@@ -1819,7 +1795,8 @@ postcore_initcall(qrtr_proto_init); -+ -+ static void __exit qrtr_proto_fini(void) -+ { -+- rtnl_unregister(PF_QIPCRTR, RTM_NEWADDR); -++ cancel_delayed_work_sync(&qrtr_ns_work); -++ qrtr_ns_remove(); -+ sock_unregister(qrtr_family.family); -+ proto_unregister(&qrtr_proto); -+ } -+Index: linux-4.4.60/net/qrtr/qrtr.h -+=================================================================== -+--- linux-4.4.60.orig/net/qrtr/qrtr.h -++++ linux-4.4.60/net/qrtr/qrtr.h -+@@ -33,4 +33,9 @@ void qrtr_endpoint_unregister(struct qrt -+ int qrtr_endpoint_post(struct qrtr_endpoint *ep, const void *data, size_t len); -+ -+ int qrtr_peek_pkt_size(const void *data); -++ -++void qrtr_ns_init(void); -++ -++void qrtr_ns_remove(void); -++ -+ #endif -+Index: linux-4.4.60/net/qrtr/Makefile -+=================================================================== -+--- linux-4.4.60.orig/net/qrtr/Makefile -++++ linux-4.4.60/net/qrtr/Makefile -+@@ -1,4 +1,4 @@ -+-obj-$(CONFIG_QRTR) := qrtr.o -++obj-$(CONFIG_QRTR) := qrtr.o ns.o -+ -+ obj-$(CONFIG_QRTR_SMD) += qrtr-smd.o -+ qrtr-smd-y := smd.o -+Index: linux-4.4.60/include/trace/events/qrtr.h -+=================================================================== -+--- /dev/null -++++ linux-4.4.60/include/trace/events/qrtr.h -+@@ -0,0 +1,115 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++#undef TRACE_SYSTEM -++#define TRACE_SYSTEM qrtr -++ -++#if !defined(_TRACE_QRTR_H) || defined(TRACE_HEADER_MULTI_READ) -++#define _TRACE_QRTR_H -++ -++#include -++#include -++ -++TRACE_EVENT(qrtr_ns_service_announce_new, -++ -++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), -++ -++ TP_ARGS(service, instance, node, port), -++ -++ TP_STRUCT__entry( -++ __field(__le32, service) -++ __field(__le32, instance) -++ __field(__le32, node) -++ __field(__le32, port) -++ ), -++ -++ TP_fast_assign( -++ __entry->service = service; -++ __entry->instance = instance; -++ __entry->node = node; -++ __entry->port = port; -++ ), -++ -++ TP_printk("advertising new server [%d:%x]@[%d:%d]", -++ __entry->service, __entry->instance, __entry->node, -++ __entry->port -++ ) -++); -++ -++TRACE_EVENT(qrtr_ns_service_announce_del, -++ -++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), -++ -++ TP_ARGS(service, instance, node, port), -++ -++ TP_STRUCT__entry( -++ __field(__le32, service) -++ __field(__le32, instance) -++ __field(__le32, node) -++ __field(__le32, port) -++ ), -++ -++ TP_fast_assign( -++ __entry->service = service; -++ __entry->instance = instance; -++ __entry->node = node; -++ __entry->port = port; -++ ), -++ -++ TP_printk("advertising removal of server [%d:%x]@[%d:%d]", -++ __entry->service, __entry->instance, __entry->node, -++ __entry->port -++ ) -++); -++ -++TRACE_EVENT(qrtr_ns_server_add, -++ -++ TP_PROTO(__le32 service, __le32 instance, __le32 node, __le32 port), -++ -++ TP_ARGS(service, instance, node, port), -++ -++ TP_STRUCT__entry( -++ __field(__le32, service) -++ __field(__le32, instance) -++ __field(__le32, node) -++ __field(__le32, port) -++ ), -++ -++ TP_fast_assign( -++ __entry->service = service; -++ __entry->instance = instance; -++ __entry->node = node; -++ __entry->port = port; -++ ), -++ -++ TP_printk("add server [%d:%x]@[%d:%d]", -++ __entry->service, __entry->instance, __entry->node, -++ __entry->port -++ ) -++); -++ -++TRACE_EVENT(qrtr_ns_message, -++ -++ TP_PROTO(const char * const ctrl_pkt_str, __u32 sq_node, __u32 sq_port), -++ -++ TP_ARGS(ctrl_pkt_str, sq_node, sq_port), -++ -++ TP_STRUCT__entry( -++ __string(ctrl_pkt_str, ctrl_pkt_str) -++ __field(__u32, sq_node) -++ __field(__u32, sq_port) -++ ), -++ -++ TP_fast_assign( -++ __assign_str(ctrl_pkt_str, ctrl_pkt_str); -++ __entry->sq_node = sq_node; -++ __entry->sq_port = sq_port; -++ ), -++ -++ TP_printk("%s from %d:%d", -++ __get_str(ctrl_pkt_str), __entry->sq_node, __entry->sq_port -++ ) -++); -++ -++#endif /* _TRACE_QRTR_H */ -++ -++/* This part must be outside protection */ -++#include -diff --git a/target/linux/ipq807x/patches/101-squashfs.patch b/target/linux/ipq807x/patches/101-squashfs.patch -new file mode 100644 -index 0000000000..5744a2a5bf ---- /dev/null -+++ b/target/linux/ipq807x/patches/101-squashfs.patch -@@ -0,0 +1,16 @@ -+Index: linux-4.4.60/fs/squashfs/xz_wrapper.c -+=================================================================== -+--- linux-4.4.60.orig/fs/squashfs/xz_wrapper.c -++++ linux-4.4.60/fs/squashfs/xz_wrapper.c -+@@ -40,10 +40,8 @@ struct squashfs_xz { -+ }; -+ -+ struct disk_comp_opts { -+- __le32 flags; -+- __le16 bit_opts; -+- __le16 fb; -+ __le32 dictionary_size; -++ __le32 flags; -+ }; -+ -+ struct comp_opts { -diff --git a/target/linux/ipq807x/patches/102-cig-wf188.patch b/target/linux/ipq807x/patches/102-cig-wf188.patch -new file mode 100644 -index 0000000000..05c736dfd7 ---- /dev/null -+++ b/target/linux/ipq807x/patches/102-cig-wf188.patch -@@ -0,0 +1,869 @@ -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/Makefile -+=================================================================== -+--- linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1.orig/arch/arm64/boot/dts/qcom/Makefile -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/Makefile -+@@ -30,6 +30,8 @@ dtb-$(CONFIG_ARCH_QCOM) += \ -+ qcom-ipq6018-cp01-c4.dtb \ -+ qcom-ipq6018-cp02-c1.dtb \ -+ qcom-ipq6018-cp03-c1.dtb \ -++ qcom-ipq6018-cig-wf188.dts \ -++ qcom-ipq6018-cig-wf188n.dts \ -+ qcom-ipq807x-hk01.dtb \ -+ qcom-ipq807x-hk01.c2.dtb \ -+ qcom-ipq807x-hk01.c3.dtb \ -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188.dts -+@@ -0,0 +1,423 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++ -++#include "qcom-ipq6018.dtsi" -++#include "qcom-ipq6018-rpm-regulator.dtsi" -++#include "qcom-ipq6018-cpr-regulator.dtsi" -++#include "qcom-ipq6018-cp-cpu.dtsi" -++#include -++#include -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "Cigtech WF-188"; -++ compatible = "cig,wf188", "qcom,ipq6018-cp03", "qcom,ipq6018"; -++ interrupt-parent = <&intc>; -++ -++ aliases { -++ serial0 = &blsp1_uart3; -++ serial1 = &blsp1_uart2; -++ -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -++ bootargs-append = " swiotlb=1 coherent_pool=2M"; -++ }; -++ -++ /* -++ * +=========+==============+========================+ -++ * | | | | -++ * | Region | Start Offset | Size | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | | | | -++ * | | | | -++ * | | | | -++ * | | | | -++ * | Linux | 0x41000000 | 139MB | -++ * | | | | -++ * | | | | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | TZ App | 0x49B00000 | 6MB | -++ * +--------+--------------+-------------------------+ -++ * -++ * From the available 145 MB for Linux in the first 256 MB, -++ * we are reserving 6 MB for TZAPP. -++ * -++ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -++ * for memory layout. -++ */ -++ -++/* TZAPP is enabled only in default memory profile */ -++#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -++ reserved-memory { -++ tzapp:tzapp@49B00000 { /* TZAPPS */ -++ no-map; -++ reg = <0x0 0x49B00000 0x0 0x00600000>; -++ }; -++ }; -++#endif -++}; -++ -++&tlmm { -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio44", "gpio45"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio22"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio64"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio75"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ leds_pins: leds_pins { -++ led_5g { -++ pins = "gpio25"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_2g { -++ pins = "gpio24"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_eth { -++ pins = "gpio18"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_pwr { -++ pins = "gpio16"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ uart2_pins: uart2_pins { -++ mux { -++ pins = "gpio57", "gpio58"; -++ function = "blsp4_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 75 0>; -++ status = "ok"; -++ phy0: ethernet-phy@0 { -++ reg = <3>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <4>; -++ }; -++ }; -++ -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x08>; /* lan port bitmap */ -++ switch_wan_bmp = <0x10>; /* wan port bitmap */ -++ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -++ qcom,port_phyinfo { -++ port@3 { -++ port_id = <3>; -++ phy_address = <4>; -++ }; -++ port@4 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <3>; -++ reg = <0x3a001400 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001600 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ -++ }; -++ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ wps { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 22 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ -++ led@25 { -++ label = "green:wifi5"; -++ gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:5g"; -++ default-state = "off"; -++ }; -++ led@24 { -++ label = "green:wifi2"; -++ gpios = <&tlmm 24 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:2g"; -++ default-state = "off"; -++ }; -++ led@18 { -++ label = "green:wan"; -++ gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:eth"; -++ default-state = "off"; -++ }; -++ led_power: led@16 { -++ label = "green:power"; -++ gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:power"; -++ default-state = "off"; -++ }; -++ }; -++ -++ gpio-watchdog { -++ compatible = "linux,wdt-gpio"; -++ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -++ hw_algo = "toggle"; -++ hw_margin_ms = <5000>; -++ always-running; -++ }; -++}; -++ -++&blsp1_uart3 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&blsp1_uart2 { -++ pinctrl-0 = <&uart2_pins>; -++ pinctrl-names = "default"; -++ dmas = <&blsp_dma 2>, -++ <&blsp_dma 3>; -++ dma-names = "tx", "rx"; -++ status = "ok"; -++}; -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "disabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&usb3 { -++ status = "ok"; -++}; -++ -++&nss_crypto { -++ status = "ok"; -++}; -++ -++&cpu0_opp_table { -++ compatible = "operating-points-v2"; -++ opp-shared; -++ opp03 { -++ opp-hz = /bits/ 64 <1200000000>; -++ opp-microvolt = <3>; -++ clock-latency-ns = <200000>; -++ }; -++ /delete-node/ opp04; -++ /delete-node/ opp05; -++ /delete-node/ opp06; -++}; -++ -++/* TZAPP is enabled in default memory profile only */ -++#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -++&qseecom { -++ mem-start = <0x49B00000>; -++ mem-size = <0x600000>; -++ status = "ok"; -++}; -++#endif -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq6018-cig-wf188n.dts -+@@ -0,0 +1,423 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++ -++#include "qcom-ipq6018.dtsi" -++#include "qcom-ipq6018-rpm-regulator.dtsi" -++#include "qcom-ipq6018-cpr-regulator.dtsi" -++#include "qcom-ipq6018-cp-cpu.dtsi" -++#include -++#include -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "Cigtech WF-188n"; -++ compatible = "cig,wf188n", "qcom,ipq6018-cp03", "qcom,ipq6018"; -++ interrupt-parent = <&intc>; -++ -++ aliases { -++ serial0 = &blsp1_uart3; -++ serial1 = &blsp1_uart2; -++ -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -++ bootargs-append = " swiotlb=1 coherent_pool=2M"; -++ }; -++ -++ /* -++ * +=========+==============+========================+ -++ * | | | | -++ * | Region | Start Offset | Size | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | | | | -++ * | | | | -++ * | | | | -++ * | | | | -++ * | Linux | 0x41000000 | 139MB | -++ * | | | | -++ * | | | | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | TZ App | 0x49B00000 | 6MB | -++ * +--------+--------------+-------------------------+ -++ * -++ * From the available 145 MB for Linux in the first 256 MB, -++ * we are reserving 6 MB for TZAPP. -++ * -++ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -++ * for memory layout. -++ */ -++ -++/* TZAPP is enabled only in default memory profile */ -++#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -++ reserved-memory { -++ tzapp:tzapp@49B00000 { /* TZAPPS */ -++ no-map; -++ reg = <0x0 0x49B00000 0x0 0x00600000>; -++ }; -++ }; -++#endif -++}; -++ -++&tlmm { -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio44", "gpio45"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio22"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio64"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio75"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ leds_pins: leds_pins { -++ led_5g { -++ pins = "gpio25"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_2g { -++ pins = "gpio24"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_eth { -++ pins = "gpio18"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_pwr { -++ pins = "gpio16"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ uart2_pins: uart2_pins { -++ mux { -++ pins = "gpio57", "gpio58"; -++ function = "blsp4_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 75 0>; -++ status = "ok"; -++ phy0: ethernet-phy@0 { -++ reg = <3>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <4>; -++ }; -++ }; -++ -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x08>; /* lan port bitmap */ -++ switch_wan_bmp = <0x10>; /* wan port bitmap */ -++ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -++ qcom,port_phyinfo { -++ port@3 { -++ port_id = <3>; -++ phy_address = <4>; -++ }; -++ port@4 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <3>; -++ reg = <0x3a001400 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001600 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ -++ }; -++ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ wps { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 22 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ -++ led@25 { -++ label = "green:wifi5"; -++ gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:5g"; -++ default-state = "off"; -++ }; -++ led@24 { -++ label = "green:wifi2"; -++ gpios = <&tlmm 24 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:2g"; -++ default-state = "off"; -++ }; -++ led@18 { -++ label = "green:wan"; -++ gpios = <&tlmm 18 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:eth"; -++ default-state = "off"; -++ }; -++ led_power: led@16 { -++ label = "green:power"; -++ gpios = <&tlmm 16 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:power"; -++ default-state = "on"; -++ }; -++ }; -++ -++ gpio-watchdog { -++ compatible = "linux,wdt-gpio"; -++ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -++ hw_algo = "toggle"; -++ hw_margin_ms = <5000>; -++ always-running; -++ }; -++}; -++ -++&blsp1_uart3 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&blsp1_uart2 { -++ pinctrl-0 = <&uart2_pins>; -++ pinctrl-names = "default"; -++ dmas = <&blsp_dma 2>, -++ <&blsp_dma 3>; -++ dma-names = "tx", "rx"; -++ status = "ok"; -++}; -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&usb3 { -++ status = "ok"; -++}; -++ -++&nss_crypto { -++ status = "ok"; -++}; -++ -++&cpu0_opp_table { -++ compatible = "operating-points-v2"; -++ opp-shared; -++ opp03 { -++ opp-hz = /bits/ 64 <1200000000>; -++ opp-microvolt = <3>; -++ clock-latency-ns = <200000>; -++ }; -++ /delete-node/ opp04; -++ /delete-node/ opp05; -++ /delete-node/ opp06; -++}; -++ -++/* TZAPP is enabled in default memory profile only */ -++#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -++&qseecom { -++ mem-start = <0x49B00000>; -++ mem-size = <0x600000>; -++ status = "ok"; -++}; -++#endif -diff --git a/target/linux/ipq807x/patches/103-sercomm-wallaby.patch b/target/linux/ipq807x/patches/103-sercomm-wallaby.patch -new file mode 100644 -index 0000000000..c6d12fe963 ---- /dev/null -+++ b/target/linux/ipq807x/patches/103-sercomm-wallaby.patch -@@ -0,0 +1,816 @@ -+Index: linux-4.4.60-qsdk/arch/arm64/boot/dts/qcom/Makefile -+=================================================================== -+--- linux-4.4.60-qsdk.orig/arch/arm64/boot/dts/qcom/Makefile -++++ linux-4.4.60-qsdk/arch/arm64/boot/dts/qcom/Makefile -+@@ -30,6 +30,7 @@ dtb-$(CONFIG_ARCH_QCOM) += \ -+ qcom-ipq807x-ac04.dtb \ -+ qcom-ipq807x-oak02.dtb \ -+ qcom-ipq807x-oak03.dtb \ -++ qcom-ipq807x-sercomm-wallaby.dtb \ -+ qcom-ipq807x-db.hk01.dtb \ -+ qcom-ipq807x-db.hk02.dtb -+ -+Index: linux-4.4.60-qsdk/arch/arm64/boot/dts/qcom/qcom-ipq807x-sercomm-wallaby.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk/arch/arm64/boot/dts/qcom/qcom-ipq807x-sercomm-wallaby.dts -+@@ -0,0 +1,799 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++#include "qcom-ipq807x-soc.dtsi" -++#include "qcom-ipq807x-hk-cpu.dtsi" -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "Sercomm Wallaby"; -++ compatible = "sercomm,wallaby", "qcom,ipq807x-hk09", "qcom,ipq807x"; -++ qcom,msm-id = <0x156 0x0>; -++ interrupt-parent = <&intc>; -++ qcom,board-id = <0x8 0x0>; -++ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -++ -++ aliases { -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet3 = "/soc/dp4"; -++ ethernet4 = "/soc/dp5"; -++ -++ led-boot = &led_green; -++ led-failsafe = &led_green; -++ led-running = &led_green; -++ led-upgrade = &led_blue; -++ }; -++}; -++ -++&tlmm { -++ pinctrl-0 = <&btcoex_pins>; -++ pinctrl-names = "default"; -++ -++ btcoex_pins: btcoex_pins { -++ mux_0 { -++ pins = "gpio64"; -++ function = "pta1_1"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "pta1_2"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_2 { -++ pins = "gpio66"; -++ function = "pta1_0"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio68"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio69"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio25"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ mux_3 { -++ pins = "gpio44"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio23", "gpio24"; -++ function = "blsp4_uart1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_3_pins: spi_3_pins { -++ mux { -++ pins = "gpio50", "gpio52", "gpio53"; -++ function = "blsp3_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ spi_cs { -++ pins = "gpio22"; -++ function = "blsp3_spi2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ quartz_interrupt { -++ pins = "gpio47"; -++ function = "gpio"; -++ input; -++ bias-disable; -++ }; -++ quartz_reset { -++ pins = "gpio21"; -++ function = "gpio"; -++ output-low; -++ bias-disable; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio46", "gpio47", "gpio48", "gpio49"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ reset_button { -++ pins = "gpio33"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ led_pins: led_pins { -++ led_wan { -++ pins = "gpio31"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ -++ led_lan { -++ pins = "gpio32"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ -++ led_blue { -++ pins = "gpio60"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ -++ led_green { -++ pins = "gpio61"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ -++ led_red { -++ pins = "gpio62"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ usb_mux_sel_pins: usb_mux_pins { -++ mux { -++ pins = "gpio27"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ pcie0_pins: pcie_pins { -++ pcie0_rst { -++ pins = "gpio58"; -++ function = "pcie0_rst"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ pcie0_wake { -++ pins = "gpio59"; -++ function = "pcie0_wake"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ button@1 { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 33 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ mdio: mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1 &tlmm 44 1>; -++ compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; -++ phy3: ethernet-phy@3 { -++ reg = <3>; -++ }; -++ phy4: ethernet-phy@4 { -++ reg = <4>; -++ }; -++ }; -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x10>; /* lan port bitmap */ -++ switch_wan_bmp = <0x20>; /* wan port bitmap */ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -++ bm_tick_mode = <0>; /* bm tick mode */ -++ tm_tick_mode = <0>; /* tm tick mode */ -++ qcom,port_phyinfo { -++ port@0 { -++ port_id = <1>; -++ phy_address = <0>; -++ }; -++ port@1 { -++ port_id = <2>; -++ phy_address = <1>; -++ }; -++ port@2 { -++ port_id = <3>; -++ phy_address = <2>; -++ }; -++ port@3 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ port@4 { -++ port_id = <5>; -++ phy_address = <4>; -++/* port_mac_sel = "QGMAC_PORT"; */ -++ }; -++/* port@5 { -++ port_id = <6>; -++ phy_address = <28>; -++ port_mac_sel = "QGMAC_PORT"; -++ };*/ -++ }; -++ port_scheduler_resource { -++ port@0 { -++ port_id = <0>; -++ ucast_queue = <0 143>; -++ mcast_queue = <256 271>; -++ l0sp = <0 35>; -++ l0cdrr = <0 47>; -++ l0edrr = <0 47>; -++ l1cdrr = <0 7>; -++ l1edrr = <0 7>; -++ }; -++ port@1 { -++ port_id = <1>; -++ ucast_queue = <144 159>; -++ mcast_queue = <272 275>; -++ l0sp = <36 39>; -++ l0cdrr = <48 63>; -++ l0edrr = <48 63>; -++ l1cdrr = <8 11>; -++ l1edrr = <8 11>; -++ }; -++ port@2 { -++ port_id = <2>; -++ ucast_queue = <160 175>; -++ mcast_queue = <276 279>; -++ l0sp = <40 43>; -++ l0cdrr = <64 79>; -++ l0edrr = <64 79>; -++ l1cdrr = <12 15>; -++ l1edrr = <12 15>; -++ }; -++ port@3 { -++ port_id = <3>; -++ ucast_queue = <176 191>; -++ mcast_queue = <280 283>; -++ l0sp = <44 47>; -++ l0cdrr = <80 95>; -++ l0edrr = <80 95>; -++ l1cdrr = <16 19>; -++ l1edrr = <16 19>; -++ }; -++ port@4 { -++ port_id = <4>; -++ ucast_queue = <192 207>; -++ mcast_queue = <284 287>; -++ l0sp = <48 51>; -++ l0cdrr = <96 111>; -++ l0edrr = <96 111>; -++ l1cdrr = <20 23>; -++ l1edrr = <20 23>; -++ }; -++ port@5 { -++ port_id = <5>; -++ ucast_queue = <208 223>; -++ mcast_queue = <288 291>; -++ l0sp = <52 55>; -++ l0cdrr = <112 127>; -++ l0edrr = <112 127>; -++ l1cdrr = <24 27>; -++ l1edrr = <24 27>; -++ }; -++ port@6 { -++ port_id = <6>; -++ ucast_queue = <224 239>; -++ mcast_queue = <292 295>; -++ l0sp = <56 59>; -++ l0cdrr = <128 143>; -++ l0edrr = <128 143>; -++ l1cdrr = <28 31>; -++ l1edrr = <28 31>; -++ }; -++ port@7 { -++ port_id = <7>; -++ ucast_queue = <240 255>; -++ mcast_queue = <296 299>; -++ l0sp = <60 63>; -++ l0cdrr = <144 159>; -++ l0edrr = <144 159>; -++ l1cdrr = <32 35>; -++ l1edrr = <32 35>; -++ }; -++ }; -++ port_scheduler_config { -++ port@0 { -++ port_id = <0>; -++ l1scheduler { -++ group@0 { -++ sp = <0 1>; /*L0 SPs*/ -++ /*cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ /*unicast queues*/ -++ ucast_queue = <0 4 8>; -++ /*multicast queues*/ -++ mcast_queue = <256 260>; -++ /*sp cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0 0>; -++ }; -++ group@1 { -++ ucast_queue = <1 5 9>; -++ mcast_queue = <257 261>; -++ cfg = <0 1 1 1 1>; -++ }; -++ group@2 { -++ ucast_queue = <2 6 10>; -++ mcast_queue = <258 262>; -++ cfg = <0 2 2 2 2>; -++ }; -++ group@3 { -++ ucast_queue = <3 7 11>; -++ mcast_queue = <259 263>; -++ cfg = <0 3 3 3 3>; -++ }; -++ }; -++ }; -++ port@1 { -++ port_id = <1>; -++ l1scheduler { -++ group@0 { -++ sp = <36>; -++ cfg = <0 8 0 8>; -++ }; -++ group@1 { -++ sp = <37>; -++ cfg = <1 9 1 9>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <144>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <272>; -++ mcast_loop_pri = <4>; -++ cfg = <36 0 48 0 48>; -++ }; -++ }; -++ }; -++ port@2 { -++ port_id = <2>; -++ l1scheduler { -++ group@0 { -++ sp = <40>; -++ cfg = <0 12 0 12>; -++ }; -++ group@1 { -++ sp = <41>; -++ cfg = <1 13 1 13>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <160>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <276>; -++ mcast_loop_pri = <4>; -++ cfg = <40 0 64 0 64>; -++ }; -++ }; -++ }; -++ port@3 { -++ port_id = <3>; -++ l1scheduler { -++ group@0 { -++ sp = <44>; -++ cfg = <0 16 0 16>; -++ }; -++ group@1 { -++ sp = <45>; -++ cfg = <1 17 1 17>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <176>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <280>; -++ mcast_loop_pri = <4>; -++ cfg = <44 0 80 0 80>; -++ }; -++ }; -++ }; -++ port@4 { -++ port_id = <4>; -++ l1scheduler { -++ group@0 { -++ sp = <48>; -++ cfg = <0 20 0 20>; -++ }; -++ group@1 { -++ sp = <49>; -++ cfg = <1 21 1 21>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <192>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <284>; -++ mcast_loop_pri = <4>; -++ cfg = <48 0 96 0 96>; -++ }; -++ }; -++ }; -++ port@5 { -++ port_id = <5>; -++ l1scheduler { -++ group@0 { -++ sp = <52>; -++ cfg = <0 24 0 24>; -++ }; -++ group@1 { -++ sp = <53>; -++ cfg = <1 25 1 25>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <208>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <288>; -++ mcast_loop_pri = <4>; -++ cfg = <52 0 112 0 112>; -++ }; -++ }; -++ }; -++ port@6 { -++ port_id = <6>; -++ l1scheduler { -++ group@0 { -++ sp = <56>; -++ cfg = <0 28 0 28>; -++ }; -++ group@1 { -++ sp = <57>; -++ cfg = <1 29 1 29>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <224>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <292>; -++ mcast_loop_pri = <4>; -++ cfg = <56 0 128 0 128>; -++ }; -++ }; -++ }; -++ port@7 { -++ port_id = <7>; -++ l1scheduler { -++ group@0 { -++ sp = <60>; -++ cfg = <0 32 0 32>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <240>; -++ mcast_queue = <296>; -++ cfg = <60 0 144 0 144>; -++ }; -++ }; -++ }; -++ }; -++ }; -++ -++ dp4 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001600 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp5 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <5>; -++ reg = <0x3a003000 0x3fff>; -++ qcom,mactype = <1>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&led_pins>; -++ pinctrl-names = "default"; -++ -++ led_wan { -++ label = "wallaby:green:wan"; -++ gpio = <&tlmm 31 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ -++ led_lan { -++ label = "wallaby:green:lan"; -++ gpio = <&tlmm 32 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ -++ led_blue: led_blue { -++ label = "wallaby:blue:status"; -++ gpio = <&tlmm 60 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ -++ led_green: led_green { -++ label = "wallaby:green:status"; -++ gpio = <&tlmm 61 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ -++ led_red { -++ label = "wallaby:red:status"; -++ gpio = <&tlmm 62 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ -++ }; -++ nss-macsec0 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x18>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++ nss-macsec1 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x1c>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++}; -++ -++&serial_blsp4 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&spi_4 { /* BLSP1 QUP3 */ -++ pinctrl-0 = <&spi_3_pins>; -++ pinctrl-names = "default"; -++ cs-select = <2>; -++ quartz-reset-gpio = <&tlmm 21 1>; -++ status = "ok"; -++ spidev3: spi@3 { -++ compatible = "qca,spidev"; -++ reg = <0>; -++ spi-max-frequency = <24000000>; -++ }; -++}; -++ -++&serial_blsp2 { -++ status = "disabled"; -++}; -++ -++&msm_imem { -++ status = "disabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&ssphy_1 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb3_0 { -++ status = "ok"; -++}; -++ -++&usb3_1 { -++ status = "ok"; -++}; -++ -++&cryptobam { -++ status = "ok"; -++}; -++ -++&crypto { -++ status = "ok"; -++}; -++ -++&i2c_0 { -++ status = "disabled"; -++}; -++ -++&i2c_1 { -++ status = "disabled"; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&qpic_lcd { -++ status = "disabled"; -++}; -++ -++&qpic_lcd_panel { -++ status = "disabled"; -++}; -++ -++&ledc { -++ status = "disabled"; -++}; -++ -++&pcie0 { -++ status = "disabled"; -++}; -++ -++&pcie1 { -++ status = "disabled"; -++}; -++ -diff --git a/target/linux/ipq807x/patches/104-wf194c.patch b/target/linux/ipq807x/patches/104-wf194c.patch -new file mode 100644 -index 0000000000..05b1b18b6c ---- /dev/null -+++ b/target/linux/ipq807x/patches/104-wf194c.patch -@@ -0,0 +1,816 @@ -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/Makefile -+=================================================================== -+--- linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1.orig/arch/arm64/boot/dts/qcom/Makefile -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/Makefile -+@@ -55,7 +55,8 @@ dtb-$(CONFIG_ARCH_QCOM) += \ -+ qcom-ipq807x-oak03.dtb \ -+ qcom-ipq807x-sercomm-wallaby.dtb \ -+ qcom-ipq807x-db.hk01.dtb \ -+- qcom-ipq807x-db.hk02.dtb -++ qcom-ipq807x-db.hk02.dtb \ -++ qcom-ipq807x-wf194c.dtb -+ endif -+ -+ always := $(dtb-y) -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/arch/arm64/boot/dts/qcom/qcom-ipq807x-wf194c.dts -+@@ -0,0 +1,751 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++#include "qcom-ipq807x-soc.dtsi" -++#include "qcom-ipq807x-audio.dtsi" -++#include "qcom-ipq807x-hk-cpu.dtsi" -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "CIG WF194C"; -++ compatible = "cig,wf194c", "qcom,ipq807x"; -++ qcom,msm-id = <0x143 0x0>, <0x158 0x0>, <0x186 0x0>, <0x188 0x0>; -++ interrupt-parent = <&intc>; -++ qcom,board-id = <0x8 0x0>; -++ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -++ -++ aliases { -++ sdhc1 = &sdhc_1; /* SDC1 eMMC slot */ -++ sdhc2 = &sdhc_2; /* SDC2 SD slot */ -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw init=/init"; -++ #ifdef __IPQ_MEM_PROFILE_256_MB__ -++ bootargs-append = " swiotlb=1"; -++ #else -++ bootargs-append = " swiotlb=1 coherent_pool=2M"; -++ #endif -++ }; -++}; -++ -++&tlmm { -++ pinctrl-0 = <&btcoex_pins>; -++ pinctrl-names = "default"; -++ -++ btcoex_pins: btcoex_pins { -++ mux_0 { -++ pins = "gpio34"; -++ function = "gpio"; -++ drive-strength = <6>; -++ bias-pull-up; -++ output-high; -++ }; -++ mux_1 { -++ pins = "gpio62"; -++ function = "gpio"; -++ drive-strength = <6>; -++ bias-pull-up; -++ output-high; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio68"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio69"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio23", "gpio24"; -++ function = "blsp4_uart1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ i2c_0_pins: i2c_0_pinmux { -++ mux { -++ pins = "gpio42", "gpio43"; -++ function = "blsp1_i2c"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_8 { -++ pins = "gpio16"; -++ function = "qpic_pad8"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ qpic_pad { -++ pins = "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", -++ "gpio9", "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ sd_pins: sd_pins { -++ mux { -++ pins = "gpio63"; -++ function = "sd_card"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio48", "gpio49"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ leds_pins: leds_pinmux { -++ led1_r { -++ pins = "gpio54"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led1_g { -++ pins = "gpio55"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led2_r { -++ pins = "gpio56"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led2_g { -++ pins = "gpio64"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio67"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ uniphy_pins: uniphy_pinmux { -++ mux_2 { -++ pins = "gpio37"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_3 { -++ pins = "gpio44"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ pwm_pins: pwm_pinmux { -++ mux_1 { -++ pins = "gpio25"; -++ function = "pwm02"; -++ drive-strength = <8>; -++ }; -++ }; -++}; -++ -++&soc { -++ pwm { -++ pinctrl-0 = <&pwm_pins>; -++ pinctrl-names = "default"; -++ used-pwm-indices = <1>, <0>, <0>, <0>; -++ status = "disabled"; -++ }; -++ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ button@1 { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 67 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++ -++ led@54 { -++ label = "wf194c:red:lan"; -++ gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ led@55 { -++ label = "wf194c:green:lan"; -++ gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>; -++ default-state = "on"; -++ }; -++ led@56 { -++ label = "wf194c:red:wan"; -++ gpios = <&tlmm 56 GPIO_ACTIVE_HIGH>; -++ default-state = "off"; -++ }; -++ led@64 { -++ label = "wf194c:green:wan"; -++ gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; -++ default-state = "on"; -++ }; -++ }; -++ mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 37 0 &tlmm 44 0>; -++ phy0: ethernet-phy@0 { -++ reg = <0xf>; /*<0>*/ -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <0xf>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <0xf>; -++ }; -++ phy3: ethernet-phy@3 { -++ reg = <3>; -++ }; -++ phy4: ethernet-phy@4 { -++ reg = <4>; -++ }; -++ phy5: ethernet-phy@5 { -++ compatible ="ethernet-phy-ieee802.3-c45"; -++ reg = <0>; -++ }; -++ }; -++ -++ ess-switch@3a000000 { -++ pinctrl-0 = <&uniphy_pins>; -++ pinctrl-names = "default"; -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x30>; /*..0x3e lan port bitmap */ -++ switch_wan_bmp = <0x40>; /* wan port bitmap */ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ -++ bm_tick_mode = <0>; /* bm tick mode */ -++ tm_tick_mode = <0>; /* tm tick mode */ -++ port_scheduler_resource { -++ port@0 { -++ port_id = <0>; -++ ucast_queue = <0 143>; -++ mcast_queue = <256 271>; -++ l0sp = <0 35>; -++ l0cdrr = <0 47>; -++ l0edrr = <0 47>; -++ l1cdrr = <0 7>; -++ l1edrr = <0 7>; -++ }; -++ port@1 { -++ port_id = <1>; -++ ucast_queue = <144 159>; -++ mcast_queue = <272 275>; -++ l0sp = <36 39>; -++ l0cdrr = <48 63>; -++ l0edrr = <48 63>; -++ l1cdrr = <8 11>; -++ l1edrr = <8 11>; -++ }; -++ port@2 { -++ port_id = <2>; -++ ucast_queue = <160 175>; -++ mcast_queue = <276 279>; -++ l0sp = <40 43>; -++ l0cdrr = <64 79>; -++ l0edrr = <64 79>; -++ l1cdrr = <12 15>; -++ l1edrr = <12 15>; -++ }; -++ port@3 { -++ port_id = <3>; -++ ucast_queue = <176 191>; -++ mcast_queue = <280 283>; -++ l0sp = <44 47>; -++ l0cdrr = <80 95>; -++ l0edrr = <80 95>; -++ l1cdrr = <16 19>; -++ l1edrr = <16 19>; -++ }; -++ port@4 { -++ port_id = <4>; -++ ucast_queue = <192 207>; -++ mcast_queue = <284 287>; -++ l0sp = <48 51>; -++ l0cdrr = <96 111>; -++ l0edrr = <96 111>; -++ l1cdrr = <20 23>; -++ l1edrr = <20 23>; -++ }; -++ port@5 { -++ port_id = <5>; -++ ucast_queue = <208 223>; -++ mcast_queue = <288 291>; -++ l0sp = <52 55>; -++ l0cdrr = <112 127>; -++ l0edrr = <112 127>; -++ l1cdrr = <24 27>; -++ l1edrr = <24 27>; -++ }; -++ port@6 { -++ port_id = <6>; -++ ucast_queue = <224 239>; -++ mcast_queue = <292 295>; -++ l0sp = <56 59>; -++ l0cdrr = <128 143>; -++ l0edrr = <128 143>; -++ l1cdrr = <28 31>; -++ l1edrr = <28 31>; -++ }; -++ port@7 { -++ port_id = <7>; -++ ucast_queue = <240 255>; -++ mcast_queue = <296 299>; -++ l0sp = <60 63>; -++ l0cdrr = <144 159>; -++ l0edrr = <144 159>; -++ l1cdrr = <32 35>; -++ l1edrr = <32 35>; -++ }; -++ }; -++ port_scheduler_config { -++ port@0 { -++ port_id = <0>; -++ l1scheduler { -++ group@0 { -++ sp = <0 1>; /*L0 SPs*/ -++ /*cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ /*unicast queues*/ -++ ucast_queue = <0 4 8>; -++ /*multicast queues*/ -++ mcast_queue = <256 260>; -++ /*sp cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0 0>; -++ }; -++ group@1 { -++ ucast_queue = <1 5 9>; -++ mcast_queue = <257 261>; -++ cfg = <0 1 1 1 1>; -++ }; -++ group@2 { -++ ucast_queue = <2 6 10>; -++ mcast_queue = <258 262>; -++ cfg = <0 2 2 2 2>; -++ }; -++ group@3 { -++ ucast_queue = <3 7 11>; -++ mcast_queue = <259 263>; -++ cfg = <0 3 3 3 3>; -++ }; -++ }; -++ }; -++ port@1 { -++ port_id = <1>; -++ l1scheduler { -++ group@0 { -++ sp = <36>; -++ cfg = <0 8 0 8>; -++ }; -++ group@1 { -++ sp = <37>; -++ cfg = <1 9 1 9>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <144>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <272>; -++ mcast_loop_pri = <4>; -++ cfg = <36 0 48 0 48>; -++ }; -++ }; -++ }; -++ port@2 { -++ port_id = <2>; -++ l1scheduler { -++ group@0 { -++ sp = <40>; -++ cfg = <0 12 0 12>; -++ }; -++ group@1 { -++ sp = <41>; -++ cfg = <1 13 1 13>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <160>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <276>; -++ mcast_loop_pri = <4>; -++ cfg = <40 0 64 0 64>; -++ }; -++ }; -++ }; -++ port@3 { -++ port_id = <3>; -++ l1scheduler { -++ group@0 { -++ sp = <44>; -++ cfg = <0 16 0 16>; -++ }; -++ group@1 { -++ sp = <45>; -++ cfg = <1 17 1 17>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <176>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <280>; -++ mcast_loop_pri = <4>; -++ cfg = <44 0 80 0 80>; -++ }; -++ }; -++ }; -++ port@4 { -++ port_id = <4>; -++ l1scheduler { -++ group@0 { -++ sp = <48>; -++ cfg = <0 20 0 20>; -++ }; -++ group@1 { -++ sp = <49>; -++ cfg = <1 21 1 21>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <192>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <284>; -++ mcast_loop_pri = <4>; -++ cfg = <48 0 96 0 96>; -++ }; -++ }; -++ }; -++ port@5 { -++ port_id = <5>; -++ l1scheduler { -++ group@0 { -++ sp = <52>; -++ cfg = <0 24 0 24>; -++ }; -++ group@1 { -++ sp = <53>; -++ cfg = <1 25 1 25>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <208>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <288>; -++ mcast_loop_pri = <4>; -++ cfg = <52 0 112 0 112>; -++ }; -++ }; -++ }; -++ port@6 { -++ port_id = <6>; -++ l1scheduler { -++ group@0 { -++ sp = <56>; -++ cfg = <0 28 0 28>; -++ }; -++ group@1 { -++ sp = <57>; -++ cfg = <1 29 1 29>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <224>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <292>; -++ mcast_loop_pri = <4>; -++ cfg = <56 0 128 0 128>; -++ }; -++ }; -++ }; -++ port@7 { -++ port_id = <7>; -++ l1scheduler { -++ group@0 { -++ sp = <60>; -++ cfg = <0 32 0 32>; -++ }; -++ group@1 { -++ sp = <61>; -++ cfg = <1 33 1 33>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <240>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <296>; -++ cfg = <60 0 144 0 144>; -++ }; -++ }; -++ }; -++ }; -++ }; -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001600 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ }; -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <6>; -++ reg = <0x3a007000 0x3fff>; -++ qcom,mactype = <1>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <0>; -++ phy-mode = "sgmii"; -++ }; -++}; -++ -++&serial_blsp4 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&serial_blsp2 { -++ pinctrl-0 = <&hsuart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&msm_imem { -++ status = "enabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&ssphy_1 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb3_0 { -++ status = "ok"; -++}; -++ -++&usb3_1 { -++ status = "ok"; -++}; -++ -++&cryptobam { -++ status = "ok"; -++}; -++ -++&crypto { -++ status = "ok"; -++}; -++ -++&i2c_0 { -++ pinctrl-0 = <&i2c_0_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&i2c_1 { -++ status = "disabled"; -++}; -++ -++&sdhc_1 { -++ qcom,clk-rates = <400000 25000000 50000000 100000000 \ -++ 192000000 384000000>; -++ qcom,bus-speed-mode = "HS400_1p8v", "HS200_1p8v", "DDR_1p8v"; -++ qcom,nonremovable; -++ status = "ok"; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&sdhc_2 { -++ qcom,clk-rates = <400000 25000000 50000000 100000000 \ -++ 192000000>; -++ qcom,bus-speed-mode = "HS200_1p8v", "DDR_1p8v"; -++ pinctrl-0 = <&sd_pins>; -++ pinctrl-names = "default"; -++ cd-gpios = <&tlmm 63 1>; -++ sd-ldo-gpios = <&tlmm 21 0>; -++ vqmmc-supply = <&ldo11>; -++ status = "ok"; -++}; -++ -++&qpic_lcd { -++ status = "ok"; -++}; -++ -++&qpic_lcd_panel { -++ status = "ok"; -++}; -+Index: linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/drivers/net/phy/aquantia.c -+=================================================================== -+--- linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1.orig/drivers/net/phy/aquantia.c -++++ linux-4.4.60-qsdk-ad8f8efb2edcd35cdb130466cfc1923c37ef7ec1/drivers/net/phy/aquantia.c -+@@ -32,6 +32,7 @@ -+ #define PHY_ID_AQR112 0x03a1b660 -+ #define PHY_ID_AQR113C 0x31c31C10 -+ #define PHY_ID_AQR112C 0x03a1b792 -++#define PHY_ID_AQR114C 0x31c31C22 -+ -+ #define AQ_PHY_MAX_VALID_MMD_REG 0xff01 -+ #define AQ_PHY_MAX_INVALID_MMD_REG 0xffff -+@@ -756,6 +757,25 @@ static struct phy_driver aquantia_driver -+ .update_link = aquantia_update_link, -+ .driver = { .owner = THIS_MODULE,}, -+ }, -++{ -++ .phy_id = PHY_ID_AQR114C, -++ .phy_id_mask = 0xfffffff0, -++ .name = "Aquantia AQR114C", -++ .features = PHY_AQUANTIA_FEATURES, -++ .flags = PHY_HAS_INTERRUPT, -++ .probe = aquantia_phy_probe, -++ .soft_reset = aquantia_soft_reset, -++ .config_init = aquantia_config_init, -++ .aneg_done = aquantia_aneg_done, -++ .config_aneg = aquantia_config_aneg, -++ .config_intr = aquantia_config_intr, -++ .ack_interrupt = aquantia_ack_interrupt, -++ .read_status = aquantia_read_status, -++ .suspend = aquantia_suspend, -++ .resume = aquantia_resume, -++ .update_link = aquantia_update_link, -++ .driver = { .owner = THIS_MODULE,}, -++}, -+ }; -+ -+ module_phy_driver(aquantia_driver); -+@@ -773,6 +793,7 @@ static struct mdio_device_id __maybe_unu -+ { PHY_ID_AQR112, 0xfffffff0 }, -+ { PHY_ID_AQR113C, 0xfffffff0 }, -+ { PHY_ID_AQR112C, 0xfffffff0 }, -++ { PHY_ID_AQR114C, 0xfffffff0 }, -+ { } -+ }; -+ -diff --git a/target/linux/ipq807x/patches/105-fix-dtc-gcc10-build.patch b/target/linux/ipq807x/patches/105-fix-dtc-gcc10-build.patch -new file mode 100644 -index 0000000000..f91601ec59 ---- /dev/null -+++ b/target/linux/ipq807x/patches/105-fix-dtc-gcc10-build.patch -@@ -0,0 +1,11 @@ -+--- a/scripts/dtc/dtc-lexer.lex.c_shipped -++++ b/scripts/dtc/dtc-lexer.lex.c_shipped -+@@ -637,7 +637,7 @@ char *yytext; -+ #include "srcpos.h" -+ #include "dtc-parser.tab.h" -+ -+-YYLTYPE yylloc; -++extern YYLTYPE yylloc; -+ extern bool treesource_error; -+ -+ /* CAUTION: this will stop working if we ever use yyless() or yyunput() */ -diff --git a/target/linux/ipq807x/patches/106-eap101.patch b/target/linux/ipq807x/patches/106-eap101.patch -new file mode 100644 -index 0000000000..27045a040f ---- /dev/null -+++ b/target/linux/ipq807x/patches/106-eap101.patch -@@ -0,0 +1,994 @@ -+Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-edgecore.dtsi -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-edgecore.dtsi -+@@ -0,0 +1,542 @@ -++/* -++ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++ -++#include "qcom-ipq6018.dtsi" -++#include -++#include -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ compatible = "qcom,ipq6018-cp01", "qcom,ipq6018"; -++ interrupt-parent = <&intc>; -++ qcom,msm-id = <0x192 0x0>, <0x193 0x0>; -++ -++ aliases { -++ serial0 = &blsp1_uart3; -++ serial1 = &blsp1_uart2; -++ sdhc2 = &sdhc_2; -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ ethernet2 = "/soc/dp3"; -++ -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -++#ifdef __IPQ_MEM_PROFILE_256_MB__ -++ bootargs-append = " swiotlb=1"; -++#else -++ bootargs-append = " swiotlb=1 coherent_pool=2M"; -++#endif -++ }; -++ -++}; -++ -++&tlmm { -++ pinctrl-0 = <&sd_ldo_pins>; -++ pinctrl-names = "default"; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio44", "gpio45"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ sd_ldo_pins: sd_ldo_pins { -++ mux { -++ pins = "gpio66"; -++ function = "gpio"; -++ drive-strength = <2>; -++ bias-disable; -++ output-low; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ spi_1_pins: spi_1_pins { -++ mux { -++ pins = "gpio69", "gpio71", "gpio72"; -++ function = "blsp1_spi"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ spi_cs { -++ pins = "gpio70"; -++ function = "blsp1_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ quartz_interrupt { -++ pins = "gpio78"; -++ function = "gpio"; -++ input; -++ bias-disable; -++ }; -++ quartz_reset { -++ pins = "gpio79"; -++ function = "gpio"; -++ output-low; -++ bias-disable; -++ }; -++ -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ sd_pins: sd_pins { -++ mux { -++ pins = "gpio62"; -++ function = "sd_card"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ extcon_usb_pins: extcon_usb_pins { -++ mux { -++ pins = "gpio26"; -++ function = "gpio"; -++ drive-strength = <2>; -++ bias-pull-down; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio9"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ reset_button { -++ pins = "gpio19"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio64"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio75"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ mux_3 { -++ pins = "gpio77"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ leds_pins: leds_pins { -++ led_pwr { -++ pins = "gpio74"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_5g { -++ pins = "gpio35"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_2g { -++ pins = "gpio37"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_wan { -++ pins = "gpio62"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_lan1 { -++ pins = "gpio61"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_lan2 { -++ pins = "gpio63"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_bt { -++ pins = "gpio73"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio71", "gpio72", "gpio69", "gpio70"; -++ function = "blsp1_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ btcoex_pins: btcoex_pins { -++ mux_0 { -++ pins = "gpio51"; -++ function = "pta1_1"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_1 { -++ pins = "gpio53"; -++ function = "pta1_0"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_2 { -++ pins = "gpio52"; -++ function = "pta1_2"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ extcon_usb: extcon_usb { -++ pinctrl-0 = <&extcon_usb_pins>; -++ pinctrl-names = "default"; -++ id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; -++ status = "ok"; -++ }; -++ -++ mdio: mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; -++ status = "ok"; -++ phy0: ethernet-phy@0 { -++ reg = <0x1c>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <3>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <4>; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <5>; -++ reg = <0x3a001000 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <28>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001200 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp3 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <3>; -++ reg = <0x3a001400 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ -++ nss-macsec0 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x1c>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x1e>; /* lan port bitmap */ -++ switch_wan_bmp = <0x20>; /* wan port bitmap */ -++ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -++ qcom,port_phyinfo { -++ port@2 { -++ port_id = <3>; -++ phy_address = <4>; -++ }; -++ port@1 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ port@0 { -++ port_id = <5>; -++ phy_address = <0x1c>; -++ port_mac_sel = "QGMAC_PORT"; -++ }; -++ }; -++ }; -++ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ /*wps { -++ label = "wps"; -++ linux,code = ; -++ gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ };*/ -++ reset { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ -++ led_power: led@74 { -++ label = "green:power"; -++ gpios = <&tlmm 74 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led_pwr"; -++ default-state = "on"; -++ }; -++ led@35 { -++ label = "green:wifi5"; -++ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led_5g"; -++ default-state = "off"; -++ }; -++ led@37 { -++ label = "green:wifi2"; -++ gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led_2g"; -++ default-state = "off"; -++ }; -++ }; -++}; -++ -++&blsp1_uart3 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&blsp1_uart2 { -++ pinctrl-0 = <&hsuart_pins &btcoex_pins>; -++ pinctrl-names = "default"; -++ dmas = <&blsp_dma 2>, -++ <&blsp_dma 3>; -++ dma-names = "tx", "rx"; -++ status = "ok"; -++}; -++ -++&spi_1 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_1_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ quartz-reset-gpio = <&tlmm 79 1>; -++ status = "disabled"; -++ spidev1: spi@1 { -++ compatible = "qca,spidev"; -++ reg = <0>; -++ spi-max-frequency = <24000000>; -++ }; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb2 { -++ status = "ok"; -++}; -++ -++&usb3 { -++ status = "ok"; -++}; -++ -++&nss_crypto { -++ status = "ok"; -++}; -++ -++&pcie_phy { -++ status = "ok"; -++}; -++ -++&pcie0 { -++ status = "ok"; -++}; -++ -++&qpic_lcd { -++ status = "ok"; -++}; -++ -++&qpic_lcd_panel { -++ status = "ok"; -++}; -+Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-edgecore-eap101.dts -+@@ -0,0 +1,442 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++ -++#include "qcom-ipq6018.dtsi" -++#include "qcom-ipq6018-rpm-regulator.dtsi" -++#include "qcom-ipq6018-cpr-regulator.dtsi" -++#include "qcom-ipq6018-cp-cpu.dtsi" -++#include -++#include -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "EdgeCore EAP101"; -++ compatible = "edgecore,eap101", "qcom,ipq6018-cp01", "qcom,ipq6018"; -++ interrupt-parent = <&intc>; -++ -++ aliases { -++ serial0 = &blsp1_uart3; -++ serial1 = &blsp1_uart2; -++ -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ ethernet2 = "/soc/dp3"; -++ -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -++ bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; -++ }; -++ -++ /* -++ * +=========+==============+========================+ -++ * | | | | -++ * | Region | Start Offset | Size | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | | | | -++ * | | | | -++ * | | | | -++ * | | | | -++ * | Linux | 0x41000000 | 139MB | -++ * | | | | -++ * | | | | -++ * | | | | -++ * +--------+--------------+-------------------------+ -++ * | TZ App | 0x49B00000 | 6MB | -++ * +--------+--------------+-------------------------+ -++ * -++ * From the available 145 MB for Linux in the first 256 MB, -++ * we are reserving 6 MB for TZAPP. -++ * -++ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -++ * for memory layout. -++ */ -++ -++/* TZAPP is enabled only in default memory profile */ -++#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -++ reserved-memory { -++ tzapp:tzapp@49B00000 { /* TZAPPS */ -++ no-map; -++ reg = <0x0 0x49B00000 0x0 0x00600000>; -++ }; -++ }; -++#endif -++}; -++ -++&tlmm { -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio44", "gpio45"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ extcon_usb_pins: extcon_usb_pins { -++ mux { -++ pins = "gpio26"; -++ function = "gpio"; -++ drive-strength = <2>; -++ bias-pull-down; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio19"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio64"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio75"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ leds_pins: leds_pins { -++ led_pwr { -++ pins = "gpio74"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_5g { -++ pins = "gpio35"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led_2g { -++ pins = "gpio37"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ uart2_pins: uart2_pins { -++ mux { -++ pins = "gpio57", "gpio58"; -++ function = "blsp4_uart"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ extcon_usb: extcon_usb { -++ pinctrl-0 = <&extcon_usb_pins>; -++ pinctrl-names = "default"; -++ id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; -++ status = "ok"; -++ }; -++ -++ mdio: mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; -++ status = "ok"; -++ phy0: ethernet-phy@0 { -++ reg = <0x1c>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <3>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <4>; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <5>; -++ reg = <0x3a001000 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <28>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001200 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp3 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <3>; -++ reg = <0x3a001400 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x1e>; /* lan port bitmap */ -++ switch_wan_bmp = <0x20>; /* wan port bitmap */ -++ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -++ qcom,port_phyinfo { -++ port@2 { -++ port_id = <3>; -++ phy_address = <4>; -++ }; -++ port@1 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ port@0 { -++ port_id = <5>; -++ phy_address = <0x1c>; -++ port_mac_sel = "QGMAC_PORT"; -++ }; -++ }; -++ }; -++ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ wps { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ -++ led@25 { -++ label = "green:wifi5"; -++ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:5g"; -++ default-state = "off"; -++ }; -++ led@24 { -++ label = "green:wifi2"; -++ gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "wf188:green:2g"; -++ default-state = "off"; -++ }; -++ led_power: led@16 { -++ label = "green:led_pwr"; -++ gpios = <&tlmm 74 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "green:power"; -++ default-state = "off"; -++ }; -++ led@61 { -++ label = "green:lan1"; -++ gpios = <&tlmm 61 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "green:power"; -++ default-state = "off"; -++ }; -++ led@62 { -++ label = "green:wan"; -++ gpios = <&tlmm 62 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "green:power"; -++ default-state = "off"; -++ }; -++ led@63 { -++ label = "green:lan2"; -++ gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "green:power"; -++ default-state = "off"; -++ }; -++ }; -++}; -++ -++&blsp1_uart3 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&blsp1_uart2 { -++ pinctrl-0 = <&uart2_pins>; -++ pinctrl-names = "default"; -++ dmas = <&blsp_dma 2>, -++ <&blsp_dma 3>; -++ dma-names = "tx", "rx"; -++ status = "ok"; -++}; -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb2 { -++ status = "ok"; -++}; -++ -++&usb3 { -++ status = "ok"; -++}; -++ -++&nss_crypto { -++ status = "ok"; -++}; -diff --git a/target/linux/ipq807x/patches/108-log-spam.patch b/target/linux/ipq807x/patches/108-log-spam.patch -new file mode 100644 -index 0000000000..94b2a3ffd5 ---- /dev/null -+++ b/target/linux/ipq807x/patches/108-log-spam.patch -@@ -0,0 +1,37 @@ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/clk/qcom/clk-branch.c -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/clk/qcom/clk-branch.c -+@@ -75,7 +75,7 @@ static int clk_branch_wait(const struct -+ bool (check_halt)(const struct clk_branch *, bool)) -+ { -+ bool voted = br->halt_check & BRANCH_VOTED; -+- const char *name = clk_hw_get_name(&br->clkr.hw); -++ //const char *name = clk_hw_get_name(&br->clkr.hw); -+ -+ /* Skip checking halt bit if the clock is in hardware gated mode */ -+ if (clk_branch_in_hwcg_mode(br)) -+@@ -93,8 +93,8 @@ static int clk_branch_wait(const struct -+ return 0; -+ udelay(1); -+ } -+- WARN(1, "%s status stuck at 'o%s'", name, -+- enabling ? "ff" : "n"); -++/* WARN(1, "%s status stuck at 'o%s'", name, -++ enabling ? "ff" : "n");*/ -+ return -EBUSY; -+ } -+ return 0; -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/usb/phy/phy-msm-qusb.c -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/usb/phy/phy-msm-qusb.c -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/usb/phy/phy-msm-qusb.c -+@@ -491,7 +491,7 @@ static int qusb_phy_init(struct usb_phy -+ dev_err(phy->dev, "QUSB PHY PLL LOCK fails:%x\n", -+ readb_relaxed(qphy->base -+ + QUSB2PHY_PLL_STATUS)); -+- WARN_ON(1); -++ //WARN_ON(1); -+ } -+ -+ /* Set OTG VBUS Valid from HSPHY to controller */ -diff --git a/target/linux/ipq807x/patches/109-tplink.patch b/target/linux/ipq807x/patches/109-tplink.patch -new file mode 100644 -index 0000000000..3ad3ffd99c ---- /dev/null -+++ b/target/linux/ipq807x/patches/109-tplink.patch -@@ -0,0 +1,1518 @@ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex227.dts -+@@ -0,0 +1,754 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++#include "qcom-ipq807x-soc.dtsi" -++#include "qcom-ipq807x-hk-cpu.dtsi" -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "TP-Link EX227"; -++ compatible = "tplink,ex227", "qcom,ipq807x"; -++ qcom,msm-id = <0x143 0x0>; -++ interrupt-parent = <&intc>; -++ qcom,board-id = <0x8 0x0>; -++ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -++ -++ aliases { -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ -++ init=/init"; -++ bootargs-append = " swiotlb=1"; -++ }; -++}; -++ -++&tlmm { -++ pinctrl-0 = <&btcoex_pins>; -++ pinctrl-names = "default"; -++ -++ btcoex_pins: btcoex_pins { -++ mux_0 { -++ pins = "gpio64"; -++ function = "pta1_1"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "pta1_2"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_2 { -++ pins = "gpio66"; -++ function = "pta1_0"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio68"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio69"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio25"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ mux_3 { -++ pins = "gpio37"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio23", "gpio24"; -++ function = "blsp4_uart1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_3_pins: spi_3_pins { -++ mux { -++ pins = "gpio52", "gpio53"; -++ function = "blsp3_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ spi_cs { -++ pins = "gpio22"; -++ function = "blsp3_spi2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ quartz_interrupt { -++ pins = "gpio47"; -++ function = "gpio"; -++ input; -++ bias-disable; -++ }; -++ quartz_reset { -++ pins = "gpio21"; -++ function = "gpio"; -++ output-low; -++ bias-disable; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio46", "gpio47", "gpio48", "gpio49"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ /* POWER_LED, TP-Link */ -++ led_pins: led_pins { -++ led_power { -++ pins = "gpio42"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ /* BUTTON, TP-Link */ -++ button_pins: button_pins { -++ reset_button { -++ pins = "gpio50"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ usb_mux_sel_pins: usb_mux_pins { -++ mux { -++ pins = "gpio27"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ pcie0_pins: pcie_pins { -++ pcie0_rst { -++ pins = "gpio58"; -++ function = "pcie0_rst"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ pcie0_wake { -++ pins = "gpio59"; -++ function = "pcie0_wake"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++}; -++ -++&soc { -++ mdio: mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1>; -++ compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; -++ phy0: ethernet-phy@0 { -++ reg = <0>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <1>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <2>; -++ }; -++ phy3: ethernet-phy@3 { -++ reg = <3>; -++ }; -++ phy4: ethernet-phy@4 { -++ reg = <28>; -++ }; -++ phy5: ethernet-phy@5 { -++ reg = <4>; -++ }; -++ }; -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x3e>; /* lan port bitmap */ -++ switch_wan_bmp = <0x40>; /* wan port bitmap */ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ -++ bm_tick_mode = <0>; /* bm tick mode */ -++ tm_tick_mode = <0>; /* tm tick mode */ -++ qcom,port_phyinfo { -++ port@0 { -++ port_id = <1>; -++ phy_address = <0>; -++ }; -++ port@1 { -++ port_id = <2>; -++ phy_address = <1>; -++ }; -++ port@2 { -++ port_id = <3>; -++ phy_address = <2>; -++ }; -++ port@3 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ port@4 { -++ port_id = <5>; -++ phy_address = <28>; -++ port_mac_sel = "QGMAC_PORT"; -++ }; -++ port@5 { -++ port_id = <6>; -++ phy_address = <4>; -++ }; -++ }; -++ port_scheduler_resource { -++ port@0 { -++ port_id = <0>; -++ ucast_queue = <0 143>; -++ mcast_queue = <256 271>; -++ l0sp = <0 35>; -++ l0cdrr = <0 47>; -++ l0edrr = <0 47>; -++ l1cdrr = <0 7>; -++ l1edrr = <0 7>; -++ }; -++ port@1 { -++ port_id = <1>; -++ ucast_queue = <144 159>; -++ mcast_queue = <272 275>; -++ l0sp = <36 39>; -++ l0cdrr = <48 63>; -++ l0edrr = <48 63>; -++ l1cdrr = <8 11>; -++ l1edrr = <8 11>; -++ }; -++ port@2 { -++ port_id = <2>; -++ ucast_queue = <160 175>; -++ mcast_queue = <276 279>; -++ l0sp = <40 43>; -++ l0cdrr = <64 79>; -++ l0edrr = <64 79>; -++ l1cdrr = <12 15>; -++ l1edrr = <12 15>; -++ }; -++ port@3 { -++ port_id = <3>; -++ ucast_queue = <176 191>; -++ mcast_queue = <280 283>; -++ l0sp = <44 47>; -++ l0cdrr = <80 95>; -++ l0edrr = <80 95>; -++ l1cdrr = <16 19>; -++ l1edrr = <16 19>; -++ }; -++ port@4 { -++ port_id = <4>; -++ ucast_queue = <192 207>; -++ mcast_queue = <284 287>; -++ l0sp = <48 51>; -++ l0cdrr = <96 111>; -++ l0edrr = <96 111>; -++ l1cdrr = <20 23>; -++ l1edrr = <20 23>; -++ }; -++ port@5 { -++ port_id = <5>; -++ ucast_queue = <208 223>; -++ mcast_queue = <288 291>; -++ l0sp = <52 55>; -++ l0cdrr = <112 127>; -++ l0edrr = <112 127>; -++ l1cdrr = <24 27>; -++ l1edrr = <24 27>; -++ }; -++ port@6 { -++ port_id = <6>; -++ ucast_queue = <224 239>; -++ mcast_queue = <292 295>; -++ l0sp = <56 59>; -++ l0cdrr = <128 143>; -++ l0edrr = <128 143>; -++ l1cdrr = <28 31>; -++ l1edrr = <28 31>; -++ }; -++ port@7 { -++ port_id = <7>; -++ ucast_queue = <240 255>; -++ mcast_queue = <296 299>; -++ l0sp = <60 63>; -++ l0cdrr = <144 159>; -++ l0edrr = <144 159>; -++ l1cdrr = <32 35>; -++ l1edrr = <32 35>; -++ }; -++ }; -++ port_scheduler_config { -++ port@0 { -++ port_id = <0>; -++ l1scheduler { -++ group@0 { -++ sp = <0 1>; /*L0 SPs*/ -++ /*cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ /*unicast queues*/ -++ ucast_queue = <0 4 8>; -++ /*multicast queues*/ -++ mcast_queue = <256 260>; -++ /*sp cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0 0>; -++ }; -++ group@1 { -++ ucast_queue = <1 5 9>; -++ mcast_queue = <257 261>; -++ cfg = <0 1 1 1 1>; -++ }; -++ group@2 { -++ ucast_queue = <2 6 10>; -++ mcast_queue = <258 262>; -++ cfg = <0 2 2 2 2>; -++ }; -++ group@3 { -++ ucast_queue = <3 7 11>; -++ mcast_queue = <259 263>; -++ cfg = <0 3 3 3 3>; -++ }; -++ }; -++ }; -++ port@1 { -++ port_id = <1>; -++ l1scheduler { -++ group@0 { -++ sp = <36>; -++ cfg = <0 8 0 8>; -++ }; -++ group@1 { -++ sp = <37>; -++ cfg = <1 9 1 9>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <144>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <272>; -++ mcast_loop_pri = <4>; -++ cfg = <36 0 48 0 48>; -++ }; -++ }; -++ }; -++ port@2 { -++ port_id = <2>; -++ l1scheduler { -++ group@0 { -++ sp = <40>; -++ cfg = <0 12 0 12>; -++ }; -++ group@1 { -++ sp = <41>; -++ cfg = <1 13 1 13>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <160>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <276>; -++ mcast_loop_pri = <4>; -++ cfg = <40 0 64 0 64>; -++ }; -++ }; -++ }; -++ port@3 { -++ port_id = <3>; -++ l1scheduler { -++ group@0 { -++ sp = <44>; -++ cfg = <0 16 0 16>; -++ }; -++ group@1 { -++ sp = <45>; -++ cfg = <1 17 1 17>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <176>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <280>; -++ mcast_loop_pri = <4>; -++ cfg = <44 0 80 0 80>; -++ }; -++ }; -++ }; -++ port@4 { -++ port_id = <4>; -++ l1scheduler { -++ group@0 { -++ sp = <48>; -++ cfg = <0 20 0 20>; -++ }; -++ group@1 { -++ sp = <49>; -++ cfg = <1 21 1 21>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <192>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <284>; -++ mcast_loop_pri = <4>; -++ cfg = <48 0 96 0 96>; -++ }; -++ }; -++ }; -++ port@5 { -++ port_id = <5>; -++ l1scheduler { -++ group@0 { -++ sp = <52>; -++ cfg = <0 24 0 24>; -++ }; -++ group@1 { -++ sp = <53>; -++ cfg = <1 25 1 25>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <208>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <288>; -++ mcast_loop_pri = <4>; -++ cfg = <52 0 112 0 112>; -++ }; -++ }; -++ }; -++ port@6 { -++ port_id = <6>; -++ l1scheduler { -++ group@0 { -++ sp = <56>; -++ cfg = <0 28 0 28>; -++ }; -++ group@1 { -++ sp = <57>; -++ cfg = <1 29 1 29>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <224>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <292>; -++ mcast_loop_pri = <4>; -++ cfg = <56 0 128 0 128>; -++ }; -++ }; -++ }; -++ port@7 { -++ port_id = <7>; -++ l1scheduler { -++ group@0 { -++ sp = <60>; -++ cfg = <0 32 0 32>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <240>; -++ mcast_queue = <296>; -++ cfg = <60 0 144 0 144>; -++ }; -++ }; -++ }; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <6>; -++ reg = <0x3a001000 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <4>; -++ phy-mode = "sgmii"; -++ }; -++ -++ /* POWER LED, TP-Link */ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&led_pins>; -++ pinctrl-names = "default"; -++ -++ led_power: led_power { -++ label = "blue:power"; -++ gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; -++ default-state = "on"; -++ }; -++ }; -++ -++ /* BUTTON, TP-Link */ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ button@1 { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 50 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ }; -++ }; -++ -++ nss-macsec0 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x18>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++ nss-macsec1 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x1c>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++}; -++ -++&serial_blsp4 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&spi_4 { /* BLSP1 QUP3 */ -++ pinctrl-0 = <&spi_3_pins>; -++ pinctrl-names = "default"; -++ cs-select = <2>; -++ quartz-reset-gpio = <&tlmm 21 1>; -++ status = "disabled"; -++ spidev3: spi@3 { -++ compatible = "qca,spidev"; -++ reg = <0>; -++ spi-max-frequency = <24000000>; -++ }; -++}; -++ -++&serial_blsp2 { -++ status = "disabled"; -++}; -++ -++&msm_imem { -++ status = "disabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&ssphy_1 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb3_0 { -++ status = "ok"; -++}; -++ -++&usb3_1 { -++ status = "ok"; -++}; -++ -++&cryptobam { -++ status = "ok"; -++}; -++ -++&crypto { -++ status = "ok"; -++}; -++ -++&i2c_0 { -++ status = "disabled"; -++}; -++ -++&i2c_1 { -++ status = "disabled"; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&qpic_lcd { -++ status = "disabled"; -++}; -++ -++&qpic_lcd_panel { -++ status = "disabled"; -++}; -++ -++&ledc { -++ status = "disabled"; -++}; -++ -++&pcie0 { -++ status = "ok"; -++}; -++ -++&pcie1 { -++ status = "disabled"; -++}; -++ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-ex447.dts -+@@ -0,0 +1,754 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++#include "qcom-ipq807x-soc.dtsi" -++#include "qcom-ipq807x-hk-cpu.dtsi" -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "TP-Link EX447"; -++ compatible = "tplink,ex447", "qcom,ipq807x"; -++ qcom,msm-id = <0x143 0x0>; -++ interrupt-parent = <&intc>; -++ qcom,board-id = <0x8 0x0>; -++ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -++ -++ aliases { -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ }; -++ -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw \ -++ init=/init"; -++ bootargs-append = " swiotlb=1"; -++ led-boot = &led_power; -++ led-failsafe = &led_power; -++ led-running = &led_power; -++ led-upgrade = &led_power; -++ }; -++}; -++ -++&tlmm { -++ pinctrl-0 = <&btcoex_pins>; -++ pinctrl-names = "default"; -++ -++ btcoex_pins: btcoex_pins { -++ mux_0 { -++ pins = "gpio64"; -++ function = "pta1_1"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_1 { -++ pins = "gpio65"; -++ function = "pta1_2"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ mux_2 { -++ pins = "gpio66"; -++ function = "pta1_0"; -++ drive-strength = <6>; -++ bias-pull-down; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio68"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio69"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_2 { -++ pins = "gpio25"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ mux_3 { -++ pins = "gpio37"; -++ function = "gpio"; -++ bias-pull-up; -++ }; -++ }; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio23", "gpio24"; -++ function = "blsp4_uart1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_3_pins: spi_3_pins { -++ mux { -++ pins = "gpio52", "gpio53"; -++ function = "blsp3_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ spi_cs { -++ pins = "gpio22"; -++ function = "blsp3_spi2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ quartz_interrupt { -++ pins = "gpio47"; -++ function = "gpio"; -++ input; -++ bias-disable; -++ }; -++ quartz_reset { -++ pins = "gpio21"; -++ function = "gpio"; -++ output-low; -++ bias-disable; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ qpic_pad { -++ pins = "gpio1", "gpio3", "gpio4", -++ "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio46", "gpio47", "gpio48", "gpio49"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ /* POWER_LED, TP-Link */ -++ led_pins: led_pins { -++ led_power { -++ pins = "gpio42"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ /* BUTTON, TP-Link */ -++ button_pins: button_pins { -++ reset_button { -++ pins = "gpio50"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ usb_mux_sel_pins: usb_mux_pins { -++ mux { -++ pins = "gpio27"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ pcie0_pins: pcie_pins { -++ pcie0_rst { -++ pins = "gpio58"; -++ function = "pcie0_rst"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ pcie0_wake { -++ pins = "gpio59"; -++ function = "pcie0_wake"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++}; -++ -++&soc { -++ mdio: mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 37 0 &tlmm 25 1>; -++ compatible = "qcom,ipq40xx-mdio", "qcom,qca-mdio"; -++ phy0: ethernet-phy@0 { -++ reg = <0>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <1>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <2>; -++ }; -++ phy3: ethernet-phy@3 { -++ reg = <3>; -++ }; -++ phy4: ethernet-phy@4 { -++ reg = <28>; -++ }; -++ phy5: ethernet-phy@5 { -++ reg = <4>; -++ }; -++ }; -++ -++ ess-switch@3a000000 { -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x3e>; /* lan port bitmap */ -++ switch_wan_bmp = <0x40>; /* wan port bitmap */ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xf>; /* mac mode for uniphy instance2*/ -++ bm_tick_mode = <0>; /* bm tick mode */ -++ tm_tick_mode = <0>; /* tm tick mode */ -++ qcom,port_phyinfo { -++ port@0 { -++ port_id = <1>; -++ phy_address = <0>; -++ }; -++ port@1 { -++ port_id = <2>; -++ phy_address = <1>; -++ }; -++ port@2 { -++ port_id = <3>; -++ phy_address = <2>; -++ }; -++ port@3 { -++ port_id = <4>; -++ phy_address = <3>; -++ }; -++ port@4 { -++ port_id = <5>; -++ phy_address = <28>; -++ port_mac_sel = "QGMAC_PORT"; -++ }; -++ port@5 { -++ port_id = <6>; -++ phy_address = <4>; -++ }; -++ }; -++ port_scheduler_resource { -++ port@0 { -++ port_id = <0>; -++ ucast_queue = <0 143>; -++ mcast_queue = <256 271>; -++ l0sp = <0 35>; -++ l0cdrr = <0 47>; -++ l0edrr = <0 47>; -++ l1cdrr = <0 7>; -++ l1edrr = <0 7>; -++ }; -++ port@1 { -++ port_id = <1>; -++ ucast_queue = <144 159>; -++ mcast_queue = <272 275>; -++ l0sp = <36 39>; -++ l0cdrr = <48 63>; -++ l0edrr = <48 63>; -++ l1cdrr = <8 11>; -++ l1edrr = <8 11>; -++ }; -++ port@2 { -++ port_id = <2>; -++ ucast_queue = <160 175>; -++ mcast_queue = <276 279>; -++ l0sp = <40 43>; -++ l0cdrr = <64 79>; -++ l0edrr = <64 79>; -++ l1cdrr = <12 15>; -++ l1edrr = <12 15>; -++ }; -++ port@3 { -++ port_id = <3>; -++ ucast_queue = <176 191>; -++ mcast_queue = <280 283>; -++ l0sp = <44 47>; -++ l0cdrr = <80 95>; -++ l0edrr = <80 95>; -++ l1cdrr = <16 19>; -++ l1edrr = <16 19>; -++ }; -++ port@4 { -++ port_id = <4>; -++ ucast_queue = <192 207>; -++ mcast_queue = <284 287>; -++ l0sp = <48 51>; -++ l0cdrr = <96 111>; -++ l0edrr = <96 111>; -++ l1cdrr = <20 23>; -++ l1edrr = <20 23>; -++ }; -++ port@5 { -++ port_id = <5>; -++ ucast_queue = <208 223>; -++ mcast_queue = <288 291>; -++ l0sp = <52 55>; -++ l0cdrr = <112 127>; -++ l0edrr = <112 127>; -++ l1cdrr = <24 27>; -++ l1edrr = <24 27>; -++ }; -++ port@6 { -++ port_id = <6>; -++ ucast_queue = <224 239>; -++ mcast_queue = <292 295>; -++ l0sp = <56 59>; -++ l0cdrr = <128 143>; -++ l0edrr = <128 143>; -++ l1cdrr = <28 31>; -++ l1edrr = <28 31>; -++ }; -++ port@7 { -++ port_id = <7>; -++ ucast_queue = <240 255>; -++ mcast_queue = <296 299>; -++ l0sp = <60 63>; -++ l0cdrr = <144 159>; -++ l0edrr = <144 159>; -++ l1cdrr = <32 35>; -++ l1edrr = <32 35>; -++ }; -++ }; -++ port_scheduler_config { -++ port@0 { -++ port_id = <0>; -++ l1scheduler { -++ group@0 { -++ sp = <0 1>; /*L0 SPs*/ -++ /*cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ /*unicast queues*/ -++ ucast_queue = <0 4 8>; -++ /*multicast queues*/ -++ mcast_queue = <256 260>; -++ /*sp cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0 0>; -++ }; -++ group@1 { -++ ucast_queue = <1 5 9>; -++ mcast_queue = <257 261>; -++ cfg = <0 1 1 1 1>; -++ }; -++ group@2 { -++ ucast_queue = <2 6 10>; -++ mcast_queue = <258 262>; -++ cfg = <0 2 2 2 2>; -++ }; -++ group@3 { -++ ucast_queue = <3 7 11>; -++ mcast_queue = <259 263>; -++ cfg = <0 3 3 3 3>; -++ }; -++ }; -++ }; -++ port@1 { -++ port_id = <1>; -++ l1scheduler { -++ group@0 { -++ sp = <36>; -++ cfg = <0 8 0 8>; -++ }; -++ group@1 { -++ sp = <37>; -++ cfg = <1 9 1 9>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <144>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <272>; -++ mcast_loop_pri = <4>; -++ cfg = <36 0 48 0 48>; -++ }; -++ }; -++ }; -++ port@2 { -++ port_id = <2>; -++ l1scheduler { -++ group@0 { -++ sp = <40>; -++ cfg = <0 12 0 12>; -++ }; -++ group@1 { -++ sp = <41>; -++ cfg = <1 13 1 13>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <160>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <276>; -++ mcast_loop_pri = <4>; -++ cfg = <40 0 64 0 64>; -++ }; -++ }; -++ }; -++ port@3 { -++ port_id = <3>; -++ l1scheduler { -++ group@0 { -++ sp = <44>; -++ cfg = <0 16 0 16>; -++ }; -++ group@1 { -++ sp = <45>; -++ cfg = <1 17 1 17>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <176>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <280>; -++ mcast_loop_pri = <4>; -++ cfg = <44 0 80 0 80>; -++ }; -++ }; -++ }; -++ port@4 { -++ port_id = <4>; -++ l1scheduler { -++ group@0 { -++ sp = <48>; -++ cfg = <0 20 0 20>; -++ }; -++ group@1 { -++ sp = <49>; -++ cfg = <1 21 1 21>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <192>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <284>; -++ mcast_loop_pri = <4>; -++ cfg = <48 0 96 0 96>; -++ }; -++ }; -++ }; -++ port@5 { -++ port_id = <5>; -++ l1scheduler { -++ group@0 { -++ sp = <52>; -++ cfg = <0 24 0 24>; -++ }; -++ group@1 { -++ sp = <53>; -++ cfg = <1 25 1 25>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <208>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <288>; -++ mcast_loop_pri = <4>; -++ cfg = <52 0 112 0 112>; -++ }; -++ }; -++ }; -++ port@6 { -++ port_id = <6>; -++ l1scheduler { -++ group@0 { -++ sp = <56>; -++ cfg = <0 28 0 28>; -++ }; -++ group@1 { -++ sp = <57>; -++ cfg = <1 29 1 29>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <224>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <292>; -++ mcast_loop_pri = <4>; -++ cfg = <56 0 128 0 128>; -++ }; -++ }; -++ }; -++ port@7 { -++ port_id = <7>; -++ l1scheduler { -++ group@0 { -++ sp = <60>; -++ cfg = <0 32 0 32>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <240>; -++ mcast_queue = <296>; -++ cfg = <60 0 144 0 144>; -++ }; -++ }; -++ }; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <5>; -++ reg = <0x3a001000 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <28>; -++ phy-mode = "sgmii"; -++ }; -++ -++ /* POWER LED, TP-Link */ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&led_pins>; -++ pinctrl-names = "default"; -++ -++ led_power: led_power { -++ label = "power:blue"; -++ gpio = <&tlmm 42 GPIO_ACTIVE_HIGH>; -++ default-state = "on"; -++ }; -++ }; -++ -++ /* BUTTON, TP-Link */ -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ button@1 { -++ label = "reset"; -++ linux,code = ; -++ gpios = <&tlmm 50 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ }; -++ }; -++ -++ nss-macsec0 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x18>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++ nss-macsec1 { -++ compatible = "qcom,nss-macsec"; -++ phy_addr = <0x1c>; -++ phy_access_mode = <0>; -++ mdiobus = <&mdio>; -++ }; -++}; -++ -++&serial_blsp4 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&spi_4 { /* BLSP1 QUP3 */ -++ pinctrl-0 = <&spi_3_pins>; -++ pinctrl-names = "default"; -++ cs-select = <2>; -++ quartz-reset-gpio = <&tlmm 21 1>; -++ status = "disabled"; -++ spidev3: spi@3 { -++ compatible = "qca,spidev"; -++ reg = <0>; -++ spi-max-frequency = <24000000>; -++ }; -++}; -++ -++&serial_blsp2 { -++ status = "disabled"; -++}; -++ -++&msm_imem { -++ status = "disabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&ssphy_1 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb3_0 { -++ status = "ok"; -++}; -++ -++&usb3_1 { -++ status = "ok"; -++}; -++ -++&cryptobam { -++ status = "ok"; -++}; -++ -++&crypto { -++ status = "ok"; -++}; -++ -++&i2c_0 { -++ status = "disabled"; -++}; -++ -++&i2c_1 { -++ status = "disabled"; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&qpic_lcd { -++ status = "disabled"; -++}; -++ -++&qpic_lcd_panel { -++ status = "disabled"; -++}; -++ -++&ledc { -++ status = "disabled"; -++}; -++ -++&pcie0 { -++ status = "ok"; -++}; -++ -++&pcie1 { -++ status = "disabled"; -++}; -++ -diff --git a/target/linux/ipq807x/patches/110-add-esmt-nand.patch b/target/linux/ipq807x/patches/110-add-esmt-nand.patch -new file mode 100644 -index 0000000000..d47a4d0d16 ---- /dev/null -+++ b/target/linux/ipq807x/patches/110-add-esmt-nand.patch -@@ -0,0 +1,37 @@ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/mtd/nand/nand_ids.c -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/mtd/nand/nand_ids.c -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/mtd/nand/nand_ids.c -+@@ -62,6 +62,12 @@ struct nand_flash_dev nand_flash_ids[] = -+ {"TH58NYG3S0H 8G 1.8V 8-bit", -+ { .id = {0x98, 0xa3, 0x91, 0x26} }, -+ SZ_4K, SZ_1K, SZ_256K, 0, 4, 256, NAND_ECC_INFO(8, SZ_512) }, -++ -++ {"F59D2G81KA 2G 1.8V 8-bit", -++ { .id = {0xc8, 0x5a, 0x90, 0x04} }, -++ SZ_2K, SZ_256, SZ_128K, 0, 4, 128, NAND_ECC_INFO(8, SZ_512) }, -++ -++ -+ LEGACY_ID_NAND("NAND 4MiB 5V 8-bit", 0x6B, 4, SZ_8K, SP_OPTIONS), -+ LEGACY_ID_NAND("NAND 4MiB 3,3V 8-bit", 0xE3, 4, SZ_8K, SP_OPTIONS), -+ LEGACY_ID_NAND("NAND 4MiB 3,3V 8-bit", 0xE5, 4, SZ_8K, SP_OPTIONS), -+@@ -190,6 +196,7 @@ struct nand_manufacturers nand_manuf_ids -+ {NAND_MFR_SANDISK, "SanDisk"}, -+ {NAND_MFR_INTEL, "Intel"}, -+ {NAND_MFR_ATO, "ATO"}, -++ {NAND_MFR_ESMT, "ESMT"}, -+ {NAND_MFR_GIGA, "GigaDevice"}, -+ {NAND_MFR_ATO, "ATO"}, -+ {NAND_MFR_WINBOND, "Winbond"}, -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/include/linux/mtd/nand.h -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/include/linux/mtd/nand.h -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/include/linux/mtd/nand.h -+@@ -778,6 +778,7 @@ static inline struct mtd_info *nand_to_m -+ #define NAND_MFR_ATO 0x9b -+ #define NAND_MFR_WINBOND 0xef -+ #define NAND_MFR_FIDELIX 0xe5 -++#define NAND_MFR_ESMT 0xc8 -+ -+ /* The maximum expected count of bytes in the NAND ID sequence */ -+ #define NAND_MAX_ID_LEN 8 -diff --git a/target/linux/ipq807x/patches/111-eap106.patch b/target/linux/ipq807x/patches/111-eap106.patch -new file mode 100644 -index 0000000000..497742193e ---- /dev/null -+++ b/target/linux/ipq807x/patches/111-eap106.patch -@@ -0,0 +1,765 @@ -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts -+=================================================================== -+--- /dev/null -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/arch/arm64/boot/dts/qcom/qcom-ipq807x-eap106.dts -+@@ -0,0 +1,716 @@ -++/dts-v1/; -++/* -++ * Copyright (c) 2017-2018, The Linux Foundation. All rights reserved. -++ * -++ * Permission to use, copy, modify, and/or distribute this software for any -++ * purpose with or without fee is hereby granted, provided that the above -++ * copyright notice and this permission notice appear in all copies. -++ * -++ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -++ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -++ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -++ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -++ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -++ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -++ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -++ */ -++#include "qcom-ipq807x-soc.dtsi" -++#include "qcom-ipq807x-hk-cpu.dtsi" -++ -++/ { -++ #address-cells = <0x2>; -++ #size-cells = <0x2>; -++ model = "Edgecore EAP106"; -++ compatible = "edgecore,eap106", "qcom,ipq807x-hk02", "qcom,ipq807x"; -++ qcom,msm-id = <0x143 0x0>; -++ interrupt-parent = <&intc>; -++ qcom,board-id = <0x8 0x0>; -++ qcom,pmic-id = <0x0 0x0 0x0 0x0>; -++ -++ aliases { -++ /* -++ * Aliases as required by u-boot -++ * to patch MAC addresses -++ */ -++ ethernet0 = "/soc/dp1"; -++ ethernet1 = "/soc/dp2"; -++ }; -++ chosen { -++ bootargs = "console=ttyMSM0,115200,n8 root=/dev/ram0 rw init=/init"; -++ #ifdef __IPQ_MEM_PROFILE_256_MB__ -++ bootargs-append = " swiotlb=1"; -++ #else -++ bootargs-append = " swiotlb=1 coherent_pool=2M"; -++ #endif -++ }; -++}; -++ -++&tlmm { -++ leds_pins: leds_pinmux { -++ -++ led1_yellow { -++ pins = "gpio25"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led1_green { -++ pins = "gpio28"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led2_amber { -++ pins = "gpio29"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ led2_blue { -++ pins = "gpio32"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-down; -++ }; -++ }; -++ -++ mdio_pins: mdio_pinmux { -++ mux_0 { -++ pins = "gpio68"; -++ function = "mdc"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ mux_1 { -++ pins = "gpio69"; -++ function = "mdio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ uart_pins: uart_pins { -++ mux { -++ pins = "gpio23", "gpio24"; -++ function = "blsp4_uart1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ i2c_0_pins: i2c_0_pinmux { -++ mux { -++ pins = "gpio42", "gpio43"; -++ function = "blsp1_i2c"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ spi_0_pins: spi_0_pins { -++ mux { -++ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -++ function = "blsp0_spi"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ qpic_pins: qpic_pins { -++ data_0 { -++ pins = "gpio15"; -++ function = "qpic_pad0"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_1 { -++ pins = "gpio12"; -++ function = "qpic_pad1"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_2 { -++ pins = "gpio13"; -++ function = "qpic_pad2"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_3 { -++ pins = "gpio14"; -++ function = "qpic_pad3"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_4 { -++ pins = "gpio5"; -++ function = "qpic_pad4"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_5 { -++ pins = "gpio6"; -++ function = "qpic_pad5"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_6 { -++ pins = "gpio7"; -++ function = "qpic_pad6"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_7 { -++ pins = "gpio8"; -++ function = "qpic_pad7"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ data_8 { -++ pins = "gpio16"; -++ function = "qpic_pad8"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ qpic_pad { -++ pins = "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", -++ "gpio9", "gpio10", "gpio11", "gpio17"; -++ function = "qpic_pad"; -++ drive-strength = <8>; -++ bias-disable; -++ }; -++ }; -++ -++ hsuart_pins: hsuart_pins { -++ mux { -++ pins = "gpio46", "gpio47", "gpio48", "gpio49"; -++ function = "blsp2_uart"; -++ drive-strength = <8>; -++ bias-disable; -++ output-low; -++ }; -++ mux_1 { -++ pins = "gpio51"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-disable; -++ output-high; -++ }; -++ }; -++ -++ button_pins: button_pins { -++ wps_button { -++ pins = "gpio57"; -++ function = "gpio"; -++ drive-strength = <8>; -++ bias-pull-up; -++ }; -++ }; -++ -++ uniphy_pins: uniphy_pinmux { -++ mux { -++ pins = "gpio60"; -++ function = "rx2"; -++ bias-disable; -++ }; -++ }; -++ cnss_wlan_en_active: cnss_wlan_en_active { -++ mux { -++ pins = "gpio57"; -++ function = "gpio"; -++ drive-strength = <16>; -++ output-high; -++ bias-pull-up; -++ }; -++ }; -++ -++ cnss_wlan_en_sleep: cnss_wlan_en_sleep { -++ mux { -++ pins = "gpio57"; -++ function = "gpio"; -++ drive-strength = <2>; -++ output-low; -++ bias-pull-down; -++ }; -++ }; -++}; -++ -++&soc { -++ gpio_keys { -++ compatible = "gpio-keys"; -++ pinctrl-0 = <&button_pins>; -++ pinctrl-names = "default"; -++ -++ button@1 { -++ label = "wps"; -++ linux,code = ; -++ gpios = <&tlmm 57 GPIO_ACTIVE_LOW>; -++ linux,input-type = <1>; -++ debounce-interval = <60>; -++ }; -++ }; -++ -++ leds { -++ compatible = "gpio-leds"; -++ pinctrl-0 = <&leds_pins>; -++ pinctrl-names = "default"; -++ -++ led@25 { -++ label = "led1_yellow"; -++ gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led1_yellow"; -++ default-state = "off"; -++ }; -++ -++ led@28 { -++ label = "led1_green"; -++ gpios = <&tlmm 28 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led1_green"; -++ default-state = "off"; -++ }; -++ -++ led@29 { -++ label = "led2_amber"; -++ gpios = <&tlmm 29 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led2_amber"; -++ default-state = "off"; -++ }; -++ -++ led@32 { -++ label = "led2_blue"; -++ gpio = <&tlmm 32 GPIO_ACTIVE_HIGH>; -++ linux,default-trigger = "led2_blue"; -++ default-state = "off"; -++ }; -++ }; -++ -++ mdio@90000 { -++ pinctrl-0 = <&mdio_pins>; -++ pinctrl-names = "default"; -++ phy-reset-gpio = <&tlmm 37 0>; -++ phy0: ethernet-phy@0 { -++ reg = <0>; -++ }; -++ phy1: ethernet-phy@1 { -++ reg = <1>; -++ }; -++ phy2: ethernet-phy@2 { -++ reg = <2>; -++ }; -++ phy3: ethernet-phy@3 { -++ reg = <3>; -++ }; -++ phy4: ethernet-phy@4 { -++ reg = <4>; -++ }; -++ phy5: ethernet-phy@5 { -++ compatible ="ethernet-phy-ieee802.3-c45"; -++ reg = <8>; -++ }; -++ }; -++ -++ ess-switch@3a000000 { -++ pinctrl-0 = <&uniphy_pins>; -++ pinctrl-names = "default"; -++ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -++ switch_lan_bmp = <0x30>; /* lan port bitmap */ -++ switch_wan_bmp = <0x40>; /* wan port bitmap */ -++ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -++ switch_mac_mode1 = <0xff>; /* mac mode for uniphy instance1*/ -++ switch_mac_mode2 = <0xd>; /* mac mode for uniphy instance2*/ -++ bm_tick_mode = <0>; /* bm tick mode */ -++ tm_tick_mode = <0>; /* tm tick mode */ -++ port_scheduler_resource { -++ port@0 { -++ port_id = <0>; -++ ucast_queue = <0 143>; -++ mcast_queue = <256 271>; -++ l0sp = <0 35>; -++ l0cdrr = <0 47>; -++ l0edrr = <0 47>; -++ l1cdrr = <0 7>; -++ l1edrr = <0 7>; -++ }; -++ port@1 { -++ port_id = <1>; -++ ucast_queue = <144 159>; -++ mcast_queue = <272 275>; -++ l0sp = <36 39>; -++ l0cdrr = <48 63>; -++ l0edrr = <48 63>; -++ l1cdrr = <8 11>; -++ l1edrr = <8 11>; -++ }; -++ port@2 { -++ port_id = <2>; -++ ucast_queue = <160 175>; -++ mcast_queue = <276 279>; -++ l0sp = <40 43>; -++ l0cdrr = <64 79>; -++ l0edrr = <64 79>; -++ l1cdrr = <12 15>; -++ l1edrr = <12 15>; -++ }; -++ port@3 { -++ port_id = <3>; -++ ucast_queue = <176 191>; -++ mcast_queue = <280 283>; -++ l0sp = <44 47>; -++ l0cdrr = <80 95>; -++ l0edrr = <80 95>; -++ l1cdrr = <16 19>; -++ l1edrr = <16 19>; -++ }; -++ port@4 { -++ port_id = <4>; -++ ucast_queue = <192 207>; -++ mcast_queue = <284 287>; -++ l0sp = <48 51>; -++ l0cdrr = <96 111>; -++ l0edrr = <96 111>; -++ l1cdrr = <20 23>; -++ l1edrr = <20 23>; -++ }; -++ port@5 { -++ port_id = <5>; -++ ucast_queue = <208 223>; -++ mcast_queue = <288 291>; -++ l0sp = <52 55>; -++ l0cdrr = <112 127>; -++ l0edrr = <112 127>; -++ l1cdrr = <24 27>; -++ l1edrr = <24 27>; -++ }; -++ port@6 { -++ port_id = <6>; -++ ucast_queue = <224 239>; -++ mcast_queue = <292 295>; -++ l0sp = <56 59>; -++ l0cdrr = <128 143>; -++ l0edrr = <128 143>; -++ l1cdrr = <28 31>; -++ l1edrr = <28 31>; -++ }; -++ port@7 { -++ port_id = <7>; -++ ucast_queue = <240 255>; -++ mcast_queue = <296 299>; -++ l0sp = <60 63>; -++ l0cdrr = <144 159>; -++ l0edrr = <144 159>; -++ l1cdrr = <32 35>; -++ l1edrr = <32 35>; -++ }; -++ }; -++ port_scheduler_config { -++ port@0 { -++ port_id = <0>; -++ l1scheduler { -++ group@0 { -++ sp = <0 1>; /*L0 SPs*/ -++ /*cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ /*unicast queues*/ -++ ucast_queue = <0 4 8>; -++ /*multicast queues*/ -++ mcast_queue = <256 260>; -++ /*sp cpri cdrr epri edrr*/ -++ cfg = <0 0 0 0 0>; -++ }; -++ group@1 { -++ ucast_queue = <1 5 9>; -++ mcast_queue = <257 261>; -++ cfg = <0 1 1 1 1>; -++ }; -++ group@2 { -++ ucast_queue = <2 6 10>; -++ mcast_queue = <258 262>; -++ cfg = <0 2 2 2 2>; -++ }; -++ group@3 { -++ ucast_queue = <3 7 11>; -++ mcast_queue = <259 263>; -++ cfg = <0 3 3 3 3>; -++ }; -++ }; -++ }; -++ port@1 { -++ port_id = <1>; -++ l1scheduler { -++ group@0 { -++ sp = <36>; -++ cfg = <0 8 0 8>; -++ }; -++ group@1 { -++ sp = <37>; -++ cfg = <1 9 1 9>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <144>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <272>; -++ mcast_loop_pri = <4>; -++ cfg = <36 0 48 0 48>; -++ }; -++ }; -++ }; -++ port@2 { -++ port_id = <2>; -++ l1scheduler { -++ group@0 { -++ sp = <40>; -++ cfg = <0 12 0 12>; -++ }; -++ group@1 { -++ sp = <41>; -++ cfg = <1 13 1 13>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <160>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <276>; -++ mcast_loop_pri = <4>; -++ cfg = <40 0 64 0 64>; -++ }; -++ }; -++ }; -++ port@3 { -++ port_id = <3>; -++ l1scheduler { -++ group@0 { -++ sp = <44>; -++ cfg = <0 16 0 16>; -++ }; -++ group@1 { -++ sp = <45>; -++ cfg = <1 17 1 17>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <176>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <280>; -++ mcast_loop_pri = <4>; -++ cfg = <44 0 80 0 80>; -++ }; -++ }; -++ }; -++ port@4 { -++ port_id = <4>; -++ l1scheduler { -++ group@0 { -++ sp = <48>; -++ cfg = <0 20 0 20>; -++ }; -++ group@1 { -++ sp = <49>; -++ cfg = <1 21 1 21>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <192>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <284>; -++ mcast_loop_pri = <4>; -++ cfg = <48 0 96 0 96>; -++ }; -++ }; -++ }; -++ port@5 { -++ port_id = <5>; -++ l1scheduler { -++ group@0 { -++ sp = <52>; -++ cfg = <0 24 0 24>; -++ }; -++ group@1 { -++ sp = <53>; -++ cfg = <1 25 1 25>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <208>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <288>; -++ mcast_loop_pri = <4>; -++ cfg = <52 0 112 0 112>; -++ }; -++ }; -++ }; -++ port@6 { -++ port_id = <6>; -++ l1scheduler { -++ group@0 { -++ sp = <56>; -++ cfg = <0 28 0 28>; -++ }; -++ group@1 { -++ sp = <57>; -++ cfg = <1 29 1 29>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <224>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <292>; -++ mcast_loop_pri = <4>; -++ cfg = <56 0 128 0 128>; -++ }; -++ }; -++ }; -++ port@7 { -++ port_id = <7>; -++ l1scheduler { -++ group@0 { -++ sp = <60>; -++ cfg = <0 32 0 32>; -++ }; -++ group@1 { -++ sp = <61>; -++ cfg = <1 33 1 33>; -++ }; -++ }; -++ l0scheduler { -++ group@0 { -++ ucast_queue = <240>; -++ ucast_loop_pri = <16>; -++ mcast_queue = <296>; -++ cfg = <60 0 144 0 144>; -++ }; -++ }; -++ }; -++ }; -++ }; -++ -++ dp1 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <4>; -++ reg = <0x3a001600 0x200>; -++ qcom,mactype = <0>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <3>; -++ phy-mode = "sgmii"; -++ }; -++ -++ dp2 { -++ device_type = "network"; -++ compatible = "qcom,nss-dp"; -++ qcom,id = <6>; -++ reg = <0x3a007000 0x3fff>; -++ qcom,mactype = <1>; -++ local-mac-address = [000000000000]; -++ qcom,link-poll = <1>; -++ qcom,phy-mdio-addr = <8>; -++ phy-mode = "sgmii"; -++ }; -++ wifi3: wifi3@f00000 { -++ compatible = "qcom,cnss-qcn9000"; -++ wlan-en-gpio = <&tlmm 57 0>; -++ pinctrl-names = "wlan_en_active", "wlan_en_sleep"; -++ pinctrl-0 = <&cnss_wlan_en_active>; -++ pinctrl-1 = <&cnss_wlan_en_sleep>; -++ status = "disabled"; -++ }; -++}; -++ -++&serial_blsp4 { -++ pinctrl-0 = <&uart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&spi_0 { /* BLSP1 QUP1 */ -++ pinctrl-0 = <&spi_0_pins>; -++ pinctrl-names = "default"; -++ cs-select = <0>; -++ status = "ok"; -++ -++ m25p80@0 { -++ #address-cells = <1>; -++ #size-cells = <1>; -++ reg = <0>; -++ compatible = "n25q128a11"; -++ linux,modalias = "m25p80", "n25q128a11"; -++ spi-max-frequency = <50000000>; -++ use-default-sizes; -++ }; -++}; -++ -++&serial_blsp2 { -++ pinctrl-0 = <&hsuart_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&msm_imem { -++ status = "disabled"; -++}; -++ -++&ssphy_0 { -++ status = "ok"; -++}; -++ -++&qusb_phy_0 { -++ status = "ok"; -++}; -++ -++&ssphy_1 { -++ status = "ok"; -++}; -++ -++&qusb_phy_1 { -++ status = "ok"; -++}; -++ -++&usb3_0 { -++ status = "ok"; -++}; -++ -++&usb3_1 { -++ status = "ok"; -++}; -++ -++&cryptobam { -++ status = "ok"; -++}; -++ -++&crypto { -++ status = "ok"; -++}; -++ -++&i2c_0 { -++ pinctrl-0 = <&i2c_0_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&i2c_1 { -++ status = "disabled"; -++}; -++ -++&qpic_bam { -++ status = "ok"; -++}; -++ -++&nand { -++ pinctrl-0 = <&qpic_pins>; -++ pinctrl-names = "default"; -++ status = "ok"; -++}; -++ -++&pcie0 { -++ status = "disabled"; -++}; -+Index: linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/net/phy/aquantia.c -+=================================================================== -+--- linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce.orig/drivers/net/phy/aquantia.c -++++ linux-4.4.60-qsdk-10fd7d14853b7020b804acae690c8acec5d954ce/drivers/net/phy/aquantia.c -+@@ -29,6 +29,7 @@ -+ #define PHY_ID_AQR109 0x03a1b502 -+ #define PHY_ID_AQR111 0x03a1b610 -+ #define PHY_ID_AQR111B0 0x03a1b612 -++#define PHY_ID_AQR111C 0x03a1b7e2 -+ #define PHY_ID_AQR112 0x03a1b660 -+ #define PHY_ID_AQR113C 0x31c31C10 -+ #define PHY_ID_AQR112C 0x03a1b792 -+@@ -701,6 +702,23 @@ static struct phy_driver aquantia_driver -+ .driver = { .owner = THIS_MODULE,}, -+ }, -+ { -++ .phy_id = PHY_ID_AQR111C, -++ .phy_id_mask = 0xfffffff0, -++ .name = "Aquantia AQR111C", -++ .features = PHY_AQUANTIA_FEATURES, -++ .flags = PHY_HAS_INTERRUPT, -++ .probe = aquantia_phy_probe, -++ .soft_reset = aquantia_soft_reset, -++ .config_init = aquantia_config_init, -++ .aneg_done = aquantia_aneg_done, -++ .config_aneg = aquantia_config_aneg, -++ .config_intr = aquantia_config_intr, -++ .ack_interrupt = aquantia_ack_interrupt, -++ .read_status = aquantia_read_status, -++ .update_link = aquantia_update_link, -++ .driver = { .owner = THIS_MODULE,}, -++}, -++{ -+ .phy_id = PHY_ID_AQR112, -+ .phy_id_mask = 0xfffffff0, -+ .name = "Aquantia AQR112", -+@@ -790,6 +808,7 @@ static struct mdio_device_id __maybe_unu -+ { PHY_ID_AQR109, 0xfffffff0 }, -+ { PHY_ID_AQR111, 0xfffffff0 }, -+ { PHY_ID_AQR111B0, 0xfffffff0 }, -++ { PHY_ID_AQR111C, 0xfffffff0 }, -+ { PHY_ID_AQR112, 0xfffffff0 }, -+ { PHY_ID_AQR113C, 0xfffffff0 }, -+ { PHY_ID_AQR112C, 0xfffffff0 }, -diff --git a/target/linux/ipq807x/patches/112-pstore.patch b/target/linux/ipq807x/patches/112-pstore.patch -new file mode 100644 -index 0000000000..dc3960306d ---- /dev/null -+++ b/target/linux/ipq807x/patches/112-pstore.patch -@@ -0,0 +1,147 @@ -+Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+=================================================================== -+--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+@@ -92,6 +92,12 @@ -+ reg = <0x0 0x40000000 0x0 0x00800000>; -+ }; -+ -++ ramoops@4A0f0000 { -++ compatible = "ramoops"; -++ reg = <0 0x4A0f0000 0 0x10000>; -++ record-size = <0x1000>; -++ }; -++ -+ uboot@4A100000 { -+ no-map; -+ reg = <0x0 0x4A100000 0x0 0x00400000>; -+@@ -211,6 +217,12 @@ -+ reg = <0x0 0x40000000 0x0 0x01000000>; -+ }; -+ -++ ramoops@4A0f0000 { -++ compatible = "ramoops"; -++ reg = <0 0x4A0f0000 0 0x10000>; -++ record-size = <0x1000>; -++ }; -++ -+ uboot@4A100000 { -+ no-map; -+ reg = <0x0 0x4A100000 0x0 0x00400000>; -+@@ -330,6 +342,12 @@ -+ reg = <0x0 0x40000000 0x0 0x01000000>; -+ }; -+ -++ ramoops@4A0f0000 { -++ compatible = "ramoops"; -++ reg = <0 0x4A0f0000 0 0x10000>; -++ record-size = <0x1000>; -++ }; -++ -+ uboot@4A100000 { -+ no-map; -+ reg = <0x0 0x4A100000 0x0 0x00400000>; -+Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/fs/pstore/ram.c -+=================================================================== -+--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/fs/pstore/ram.c -++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/fs/pstore/ram.c -+@@ -466,15 +466,46 @@ static int ramoops_init_prz(struct devic -+ return 0; -+ } -+ -++static int ramoops_parse_dt(struct platform_device *pdev, -++ struct ramoops_platform_data *pdata) -++{ -++ struct resource *res; -++ -++ dev_dbg(&pdev->dev, "using Device Tree\n"); -++ -++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); -++ if (!res) { -++ dev_err(&pdev->dev, -++ "failed to locate DT /reserved-memory resource\n"); -++ return -EINVAL; -++ } -++ -++ pdata->mem_size = resource_size(res); -++ pdata->mem_address = res->start; -++ pdata->dump_oops = true; -++ pdata->record_size = 0x1000; -++ return 0; -++} -++ -+ static int ramoops_probe(struct platform_device *pdev) -+ { -+ struct device *dev = &pdev->dev; -+ struct ramoops_platform_data *pdata = pdev->dev.platform_data; -++ struct ramoops_platform_data pdata_local; -+ struct ramoops_context *cxt = &oops_cxt; -+ size_t dump_mem_sz; -+ phys_addr_t paddr; -+ int err = -EINVAL; -+ -++ if (dev_of_node(dev) && !pdata) { -++ pdata = &pdata_local; -++ memset(pdata, 0, sizeof(*pdata)); -++ -++ err = ramoops_parse_dt(pdev, pdata); -++ if (err < 0) -++ goto fail_out; -++ } -++ -+ /* Only a single ramoops area allowed at a time, so fail extra -+ * probes. -+ */ -+@@ -603,11 +634,17 @@ static int ramoops_remove(struct platfor -+ return 0; -+ } -+ -++static const struct of_device_id dt_match[] = { -++ { .compatible = "ramoops" }, -++ {} -++}; -++ -+ static struct platform_driver ramoops_driver = { -+ .probe = ramoops_probe, -+ .remove = ramoops_remove, -+ .driver = { -+ .name = "ramoops", -++ .of_match_table = dt_match, -+ }, -+ }; -+ -+Index: linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/drivers/of/platform.c -+=================================================================== -+--- linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016.orig/drivers/of/platform.c -++++ linux-4.4.60-qsdk-11f09717303ecd83c3a64e9efe23f25921dc1016/drivers/of/platform.c -+@@ -53,6 +53,30 @@ struct platform_device *of_find_device_b -+ } -+ EXPORT_SYMBOL(of_find_device_by_node); -+ -++static const struct of_device_id reserved_mem_matches[] = { -++ { .compatible = "ramoops" }, -++ {} -++}; -++ -++static int __init of_platform_default_populate_init(void) -++{ -++ struct device_node *node; -++ -++ if (!of_have_populated_dt()) -++ return -ENODEV; -++ -++ /* -++ * Handle certain compatibles explicitly, since we don't want to create -++ * platform_devices for every node in /reserved-memory with a -++ * "compatible", -++ */ -++ for_each_matching_node(node, reserved_mem_matches) -++ of_platform_device_create(node, NULL, NULL); -++ -++ return 0; -++} -++arch_initcall_sync(of_platform_default_populate_init); -++ -+ #ifdef CONFIG_OF_ADDRESS -+ /* -+ * The following routines scan a subtree and registers a device for -diff --git a/target/linux/ipq807x/patches/200-bpf_backport.patch b/target/linux/ipq807x/patches/200-bpf_backport.patch -new file mode 100644 -index 0000000000..4357369c29 ---- /dev/null -+++ b/target/linux/ipq807x/patches/200-bpf_backport.patch -@@ -0,0 +1,44780 @@ -+--- a/arch/arm/Kconfig -++++ b/arch/arm/Kconfig -+@@ -38,7 +38,7 @@ config ARM -+ select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 -+ select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT) -+ select HAVE_ARCH_TRACEHOOK -+- select HAVE_BPF_JIT -++ select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 -+ select HAVE_CC_STACKPROTECTOR -+ select HAVE_CONTEXT_TRACKING -+ select HAVE_C_RECORDMCOUNT -+--- a/arch/arm/net/bpf_jit_32.c -++++ b/arch/arm/net/bpf_jit_32.c -+@@ -1,13 +1,12 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* -+- * Just-In-Time compiler for BPF filters on 32bit ARM -++ * Just-In-Time compiler for eBPF filters on 32bit ARM -+ * -++ * Copyright (c) 2017 Shubham Bansal -+ * Copyright (c) 2011 Mircea Gherzan -+- * -+- * This program is free software; you can redistribute it and/or modify it -+- * under the terms of the GNU General Public License as published by the -+- * Free Software Foundation; version 2 of the License. -+ */ -+ -++#include -+ #include -+ #include -+ #include -+@@ -20,51 +19,182 @@ -+ #include -+ #include -+ #include -++#include -+ -+ #include "bpf_jit_32.h" -+ -+ /* -+- * ABI: -++ * eBPF prog stack layout: -++ * -++ * high -++ * original ARM_SP => +-----+ -++ * | | callee saved registers -++ * +-----+ <= (BPF_FP + SCRATCH_SIZE) -++ * | ... | eBPF JIT scratch space -++ * eBPF fp register => +-----+ -++ * (BPF_FP) | ... | eBPF prog stack -++ * +-----+ -++ * |RSVD | JIT scratchpad -++ * current ARM_SP => +-----+ <= (BPF_FP - STACK_SIZE + SCRATCH_SIZE) -++ * | | -++ * | ... | Function call stack -++ * | | -++ * +-----+ -++ * low -++ * -++ * The callee saved registers depends on whether frame pointers are enabled. -++ * With frame pointers (to be compliant with the ABI): -++ * -++ * high -++ * original ARM_SP => +--------------+ \ -++ * | pc | | -++ * current ARM_FP => +--------------+ } callee saved registers -++ * |r4-r9,fp,ip,lr| | -++ * +--------------+ / -++ * low -+ * -+- * r0 scratch register -+- * r4 BPF register A -+- * r5 BPF register X -+- * r6 pointer to the skb -+- * r7 skb->data -+- * r8 skb_headlen(skb) -++ * Without frame pointers: -++ * -++ * high -++ * original ARM_SP => +--------------+ -++ * | r4-r9,fp,lr | callee saved registers -++ * current ARM_FP => +--------------+ -++ * low -++ * -++ * When popping registers off the stack at the end of a BPF function, we -++ * reference them via the current ARM_FP register. -+ */ -++#define CALLEE_MASK (1 << ARM_R4 | 1 << ARM_R5 | 1 << ARM_R6 | \ -++ 1 << ARM_R7 | 1 << ARM_R8 | 1 << ARM_R9 | \ -++ 1 << ARM_FP) -++#define CALLEE_PUSH_MASK (CALLEE_MASK | 1 << ARM_LR) -++#define CALLEE_POP_MASK (CALLEE_MASK | 1 << ARM_PC) -++ -++enum { -++ /* Stack layout - these are offsets from (top of stack - 4) */ -++ BPF_R2_HI, -++ BPF_R2_LO, -++ BPF_R3_HI, -++ BPF_R3_LO, -++ BPF_R4_HI, -++ BPF_R4_LO, -++ BPF_R5_HI, -++ BPF_R5_LO, -++ BPF_R7_HI, -++ BPF_R7_LO, -++ BPF_R8_HI, -++ BPF_R8_LO, -++ BPF_R9_HI, -++ BPF_R9_LO, -++ BPF_FP_HI, -++ BPF_FP_LO, -++ BPF_TC_HI, -++ BPF_TC_LO, -++ BPF_AX_HI, -++ BPF_AX_LO, -++ /* Stack space for BPF_REG_2, BPF_REG_3, BPF_REG_4, -++ * BPF_REG_5, BPF_REG_7, BPF_REG_8, BPF_REG_9, -++ * BPF_REG_FP and Tail call counts. -++ */ -++ BPF_JIT_SCRATCH_REGS, -++}; -++ -++/* -++ * Negative "register" values indicate the register is stored on the stack -++ * and are the offset from the top of the eBPF JIT scratch space. -++ */ -++#define STACK_OFFSET(k) (-4 - (k) * 4) -++#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4) -++ -++#ifdef CONFIG_FRAME_POINTER -++#define EBPF_SCRATCH_TO_ARM_FP(x) ((x) - 4 * hweight16(CALLEE_PUSH_MASK) - 4) -++#else -++#define EBPF_SCRATCH_TO_ARM_FP(x) (x) -++#endif -+ -+-#define r_scratch ARM_R0 -+-/* r1-r3 are (also) used for the unaligned loads on the non-ARMv7 slowpath */ -+-#define r_off ARM_R1 -+-#define r_A ARM_R4 -+-#define r_X ARM_R5 -+-#define r_skb ARM_R6 -+-#define r_skb_data ARM_R7 -+-#define r_skb_hl ARM_R8 -+- -+-#define SCRATCH_SP_OFFSET 0 -+-#define SCRATCH_OFF(k) (SCRATCH_SP_OFFSET + 4 * (k)) -+- -+-#define SEEN_MEM ((1 << BPF_MEMWORDS) - 1) -+-#define SEEN_MEM_WORD(k) (1 << (k)) -+-#define SEEN_X (1 << BPF_MEMWORDS) -+-#define SEEN_CALL (1 << (BPF_MEMWORDS + 1)) -+-#define SEEN_SKB (1 << (BPF_MEMWORDS + 2)) -+-#define SEEN_DATA (1 << (BPF_MEMWORDS + 3)) -++#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) /* TEMP Register 1 */ -++#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) /* TEMP Register 2 */ -++#define TCALL_CNT (MAX_BPF_JIT_REG + 2) /* Tail Call Count */ -+ -+-#define FLAG_NEED_X_RESET (1 << 0) -+-#define FLAG_IMM_OVERFLOW (1 << 1) -++#define FLAG_IMM_OVERFLOW (1 << 0) -++ -++/* -++ * Map eBPF registers to ARM 32bit registers or stack scratch space. -++ * -++ * 1. First argument is passed using the arm 32bit registers and rest of the -++ * arguments are passed on stack scratch space. -++ * 2. First callee-saved argument is mapped to arm 32 bit registers and rest -++ * arguments are mapped to scratch space on stack. -++ * 3. We need two 64 bit temp registers to do complex operations on eBPF -++ * registers. -++ * -++ * As the eBPF registers are all 64 bit registers and arm has only 32 bit -++ * registers, we have to map each eBPF registers with two arm 32 bit regs or -++ * scratch memory space and we have to build eBPF 64 bit register from those. -++ * -++ */ -++static const s8 bpf2a32[][2] = { -++ /* return value from in-kernel function, and exit value from eBPF */ -++ [BPF_REG_0] = {ARM_R1, ARM_R0}, -++ /* arguments from eBPF program to in-kernel function */ -++ [BPF_REG_1] = {ARM_R3, ARM_R2}, -++ /* Stored on stack scratch space */ -++ [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)}, -++ [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)}, -++ [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)}, -++ [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)}, -++ /* callee saved registers that in-kernel function will preserve */ -++ [BPF_REG_6] = {ARM_R5, ARM_R4}, -++ /* Stored on stack scratch space */ -++ [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)}, -++ [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)}, -++ [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)}, -++ /* Read only Frame Pointer to access Stack */ -++ [BPF_REG_FP] = {STACK_OFFSET(BPF_FP_HI), STACK_OFFSET(BPF_FP_LO)}, -++ /* Temporary Register for internal BPF JIT, can be used -++ * for constant blindings and others. -++ */ -++ [TMP_REG_1] = {ARM_R7, ARM_R6}, -++ [TMP_REG_2] = {ARM_R9, ARM_R8}, -++ /* Tail call count. Stored on stack scratch space. */ -++ [TCALL_CNT] = {STACK_OFFSET(BPF_TC_HI), STACK_OFFSET(BPF_TC_LO)}, -++ /* temporary register for blinding constants. -++ * Stored on stack scratch space. -++ */ -++ [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)}, -++}; -++ -++#define dst_lo dst[1] -++#define dst_hi dst[0] -++#define src_lo src[1] -++#define src_hi src[0] -++ -++/* -++ * JIT Context: -++ * -++ * prog : bpf_prog -++ * idx : index of current last JITed instruction. -++ * prologue_bytes : bytes used in prologue. -++ * epilogue_offset : offset of epilogue starting. -++ * offsets : array of eBPF instruction offsets in -++ * JITed code. -++ * target : final JITed code. -++ * epilogue_bytes : no of bytes used in epilogue. -++ * imm_count : no of immediate counts used for global -++ * variables. -++ * imms : array of global variable addresses. -++ */ -+ -+ struct jit_ctx { -+- const struct bpf_prog *skf; -+- unsigned idx; -+- unsigned prologue_bytes; -+- int ret0_fp_idx; -+- u32 seen; -++ const struct bpf_prog *prog; -++ unsigned int idx; -++ unsigned int prologue_bytes; -++ unsigned int epilogue_offset; -++ unsigned int cpu_architecture; -+ u32 flags; -+ u32 *offsets; -+ u32 *target; -++ u32 stack_size; -+ #if __LINUX_ARM_ARCH__ < 7 -+ u16 epilogue_bytes; -+ u16 imm_count; -+@@ -72,68 +202,16 @@ struct jit_ctx { -+ #endif -+ }; -+ -+-int bpf_jit_enable __read_mostly; -+- -+-static inline int call_neg_helper(struct sk_buff *skb, int offset, void *ret, -+- unsigned int size) -+-{ -+- void *ptr = bpf_internal_load_pointer_neg_helper(skb, offset, size); -+- -+- if (!ptr) -+- return -EFAULT; -+- memcpy(ret, ptr, size); -+- return 0; -+-} -+- -+-static u64 jit_get_skb_b(struct sk_buff *skb, int offset) -+-{ -+- u8 ret; -+- int err; -+- -+- if (offset < 0) -+- err = call_neg_helper(skb, offset, &ret, 1); -+- else -+- err = skb_copy_bits(skb, offset, &ret, 1); -+- -+- return (u64)err << 32 | ret; -+-} -+- -+-static u64 jit_get_skb_h(struct sk_buff *skb, int offset) -+-{ -+- u16 ret; -+- int err; -+- -+- if (offset < 0) -+- err = call_neg_helper(skb, offset, &ret, 2); -+- else -+- err = skb_copy_bits(skb, offset, &ret, 2); -+- -+- return (u64)err << 32 | ntohs(ret); -+-} -+- -+-static u64 jit_get_skb_w(struct sk_buff *skb, int offset) -+-{ -+- u32 ret; -+- int err; -+- -+- if (offset < 0) -+- err = call_neg_helper(skb, offset, &ret, 4); -+- else -+- err = skb_copy_bits(skb, offset, &ret, 4); -+- -+- return (u64)err << 32 | ntohl(ret); -+-} -+- -+ /* -+ * Wrappers which handle both OABI and EABI and assures Thumb2 interworking -+ * (where the assembly routines like __aeabi_uidiv could cause problems). -+ */ -+-static u32 jit_udiv(u32 dividend, u32 divisor) -++static u32 jit_udiv32(u32 dividend, u32 divisor) -+ { -+ return dividend / divisor; -+ } -+ -+-static u32 jit_mod(u32 dividend, u32 divisor) -++static u32 jit_mod32(u32 dividend, u32 divisor) -+ { -+ return dividend % divisor; -+ } -+@@ -157,36 +235,100 @@ static inline void emit(u32 inst, struct -+ _emit(ARM_COND_AL, inst, ctx); -+ } -+ -+-static u16 saved_regs(struct jit_ctx *ctx) -++/* -++ * This is rather horrid, but necessary to convert an integer constant -++ * to an immediate operand for the opcodes, and be able to detect at -++ * build time whether the constant can't be converted (iow, usable in -++ * BUILD_BUG_ON()). -++ */ -++#define imm12val(v, s) (rol32(v, (s)) | (s) << 7) -++#define const_imm8m(x) \ -++ ({ int r; \ -++ u32 v = (x); \ -++ if (!(v & ~0x000000ff)) \ -++ r = imm12val(v, 0); \ -++ else if (!(v & ~0xc000003f)) \ -++ r = imm12val(v, 2); \ -++ else if (!(v & ~0xf000000f)) \ -++ r = imm12val(v, 4); \ -++ else if (!(v & ~0xfc000003)) \ -++ r = imm12val(v, 6); \ -++ else if (!(v & ~0xff000000)) \ -++ r = imm12val(v, 8); \ -++ else if (!(v & ~0x3fc00000)) \ -++ r = imm12val(v, 10); \ -++ else if (!(v & ~0x0ff00000)) \ -++ r = imm12val(v, 12); \ -++ else if (!(v & ~0x03fc0000)) \ -++ r = imm12val(v, 14); \ -++ else if (!(v & ~0x00ff0000)) \ -++ r = imm12val(v, 16); \ -++ else if (!(v & ~0x003fc000)) \ -++ r = imm12val(v, 18); \ -++ else if (!(v & ~0x000ff000)) \ -++ r = imm12val(v, 20); \ -++ else if (!(v & ~0x0003fc00)) \ -++ r = imm12val(v, 22); \ -++ else if (!(v & ~0x0000ff00)) \ -++ r = imm12val(v, 24); \ -++ else if (!(v & ~0x00003fc0)) \ -++ r = imm12val(v, 26); \ -++ else if (!(v & ~0x00000ff0)) \ -++ r = imm12val(v, 28); \ -++ else if (!(v & ~0x000003fc)) \ -++ r = imm12val(v, 30); \ -++ else \ -++ r = -1; \ -++ r; }) -++ -++/* -++ * Checks if immediate value can be converted to imm12(12 bits) value. -++ */ -++static int imm8m(u32 x) -+ { -+- u16 ret = 0; -++ u32 rot; -+ -+- if ((ctx->skf->len > 1) || -+- (ctx->skf->insns[0].code == (BPF_RET | BPF_A))) -+- ret |= 1 << r_A; -++ for (rot = 0; rot < 16; rot++) -++ if ((x & ~ror32(0xff, 2 * rot)) == 0) -++ return rol32(x, 2 * rot) | (rot << 8); -++ return -1; -++} -+ -+-#ifdef CONFIG_FRAME_POINTER -+- ret |= (1 << ARM_FP) | (1 << ARM_IP) | (1 << ARM_LR) | (1 << ARM_PC); -+-#else -+- if (ctx->seen & SEEN_CALL) -+- ret |= 1 << ARM_LR; -+-#endif -+- if (ctx->seen & (SEEN_DATA | SEEN_SKB)) -+- ret |= 1 << r_skb; -+- if (ctx->seen & SEEN_DATA) -+- ret |= (1 << r_skb_data) | (1 << r_skb_hl); -+- if (ctx->seen & SEEN_X) -+- ret |= 1 << r_X; -++#define imm8m(x) (__builtin_constant_p(x) ? const_imm8m(x) : imm8m(x)) -+ -+- return ret; -++static u32 arm_bpf_ldst_imm12(u32 op, u8 rt, u8 rn, s16 imm12) -++{ -++ op |= rt << 12 | rn << 16; -++ if (imm12 >= 0) -++ op |= ARM_INST_LDST__U; -++ else -++ imm12 = -imm12; -++ return op | (imm12 & ARM_INST_LDST__IMM12); -+ } -+ -+-static inline int mem_words_used(struct jit_ctx *ctx) -++static u32 arm_bpf_ldst_imm8(u32 op, u8 rt, u8 rn, s16 imm8) -+ { -+- /* yes, we do waste some stack space IF there are "holes" in the set" */ -+- return fls(ctx->seen & SEEN_MEM); -++ op |= rt << 12 | rn << 16; -++ if (imm8 >= 0) -++ op |= ARM_INST_LDST__U; -++ else -++ imm8 = -imm8; -++ return op | (imm8 & 0xf0) << 4 | (imm8 & 0x0f); -+ } -+ -++#define ARM_LDR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDR_I, rt, rn, off) -++#define ARM_LDRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_LDRB_I, rt, rn, off) -++#define ARM_LDRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRD_I, rt, rn, off) -++#define ARM_LDRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_LDRH_I, rt, rn, off) -++ -++#define ARM_STR_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STR_I, rt, rn, off) -++#define ARM_STRB_I(rt, rn, off) arm_bpf_ldst_imm12(ARM_INST_STRB_I, rt, rn, off) -++#define ARM_STRD_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRD_I, rt, rn, off) -++#define ARM_STRH_I(rt, rn, off) arm_bpf_ldst_imm8(ARM_INST_STRH_I, rt, rn, off) -++ -++/* -++ * Initializes the JIT space with undefined instructions. -++ */ -+ static void jit_fill_hole(void *area, unsigned int size) -+ { -+ u32 *ptr; -+@@ -195,88 +337,23 @@ static void jit_fill_hole(void *area, un -+ *ptr++ = __opcode_to_mem_arm(ARM_INST_UDF); -+ } -+ -+-static void build_prologue(struct jit_ctx *ctx) -+-{ -+- u16 reg_set = saved_regs(ctx); -+- u16 off; -+- -+-#ifdef CONFIG_FRAME_POINTER -+- emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); -+- emit(ARM_PUSH(reg_set), ctx); -+- emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); -++#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) -++/* EABI requires the stack to be aligned to 64-bit boundaries */ -++#define STACK_ALIGNMENT 8 -+ #else -+- if (reg_set) -+- emit(ARM_PUSH(reg_set), ctx); -++/* Stack must be aligned to 32-bit boundaries */ -++#define STACK_ALIGNMENT 4 -+ #endif -+ -+- if (ctx->seen & (SEEN_DATA | SEEN_SKB)) -+- emit(ARM_MOV_R(r_skb, ARM_R0), ctx); -+- -+- if (ctx->seen & SEEN_DATA) { -+- off = offsetof(struct sk_buff, data); -+- emit(ARM_LDR_I(r_skb_data, r_skb, off), ctx); -+- /* headlen = len - data_len */ -+- off = offsetof(struct sk_buff, len); -+- emit(ARM_LDR_I(r_skb_hl, r_skb, off), ctx); -+- off = offsetof(struct sk_buff, data_len); -+- emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); -+- emit(ARM_SUB_R(r_skb_hl, r_skb_hl, r_scratch), ctx); -+- } -+- -+- if (ctx->flags & FLAG_NEED_X_RESET) -+- emit(ARM_MOV_I(r_X, 0), ctx); -+- -+- /* do not leak kernel data to userspace */ -+- if (bpf_needs_clear_a(&ctx->skf->insns[0])) -+- emit(ARM_MOV_I(r_A, 0), ctx); -+- -+- /* stack space for the BPF_MEM words */ -+- if (ctx->seen & SEEN_MEM) -+- emit(ARM_SUB_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); -+-} -+- -+-static void build_epilogue(struct jit_ctx *ctx) -+-{ -+- u16 reg_set = saved_regs(ctx); -+- -+- if (ctx->seen & SEEN_MEM) -+- emit(ARM_ADD_I(ARM_SP, ARM_SP, mem_words_used(ctx) * 4), ctx); -+- -+- reg_set &= ~(1 << ARM_LR); -+- -+-#ifdef CONFIG_FRAME_POINTER -+- /* the first instruction of the prologue was: mov ip, sp */ -+- reg_set &= ~(1 << ARM_IP); -+- reg_set |= (1 << ARM_SP); -+- emit(ARM_LDM(ARM_SP, reg_set), ctx); -+-#else -+- if (reg_set) { -+- if (ctx->seen & SEEN_CALL) -+- reg_set |= 1 << ARM_PC; -+- emit(ARM_POP(reg_set), ctx); -+- } -+- -+- if (!(ctx->seen & SEEN_CALL)) -+- emit(ARM_BX(ARM_LR), ctx); -+-#endif -+-} -+- -+-static int16_t imm8m(u32 x) -+-{ -+- u32 rot; -+- -+- for (rot = 0; rot < 16; rot++) -+- if ((x & ~ror32(0xff, 2 * rot)) == 0) -+- return rol32(x, 2 * rot) | (rot << 8); -+- -+- return -1; -+-} -++/* total stack size used in JITed code */ -++#define _STACK_SIZE (ctx->prog->aux->stack_depth + SCRATCH_SIZE) -++#define STACK_SIZE ALIGN(_STACK_SIZE, STACK_ALIGNMENT) -+ -+ #if __LINUX_ARM_ARCH__ < 7 -+ -+ static u16 imm_offset(u32 k, struct jit_ctx *ctx) -+ { -+- unsigned i = 0, offset; -++ unsigned int i = 0, offset; -+ u16 imm; -+ -+ /* on the "fake" run we just count them (duplicates included) */ -+@@ -295,7 +372,7 @@ static u16 imm_offset(u32 k, struct jit_ -+ ctx->imms[i] = k; -+ -+ /* constants go just after the epilogue */ -+- offset = ctx->offsets[ctx->skf->len]; -++ offset = ctx->offsets[ctx->prog->len - 1] * 4; -+ offset += ctx->prologue_bytes; -+ offset += ctx->epilogue_bytes; -+ offset += i * 4; -+@@ -319,10 +396,22 @@ static u16 imm_offset(u32 k, struct jit_ -+ -+ #endif /* __LINUX_ARM_ARCH__ */ -+ -++static inline int bpf2a32_offset(int bpf_to, int bpf_from, -++ const struct jit_ctx *ctx) { -++ int to, from; -++ -++ if (ctx->target == NULL) -++ return 0; -++ to = ctx->offsets[bpf_to]; -++ from = ctx->offsets[bpf_from]; -++ -++ return to - from - 1; -++} -++ -+ /* -+ * Move an immediate that's not an imm8m to a core register. -+ */ -+-static inline void emit_mov_i_no8m(int rd, u32 val, struct jit_ctx *ctx) -++static inline void emit_mov_i_no8m(const u8 rd, u32 val, struct jit_ctx *ctx) -+ { -+ #if __LINUX_ARM_ARCH__ < 7 -+ emit(ARM_LDR_I(rd, ARM_PC, imm_offset(val, ctx)), ctx); -+@@ -333,7 +422,7 @@ static inline void emit_mov_i_no8m(int r -+ #endif -+ } -+ -+-static inline void emit_mov_i(int rd, u32 val, struct jit_ctx *ctx) -++static inline void emit_mov_i(const u8 rd, u32 val, struct jit_ctx *ctx) -+ { -+ int imm12 = imm8m(val); -+ -+@@ -343,676 +432,1508 @@ static inline void emit_mov_i(int rd, u3 -+ emit_mov_i_no8m(rd, val, ctx); -+ } -+ -+-#if __LINUX_ARM_ARCH__ < 6 -++static void emit_bx_r(u8 tgt_reg, struct jit_ctx *ctx) -++{ -++ if (elf_hwcap & HWCAP_THUMB) -++ emit(ARM_BX(tgt_reg), ctx); -++ else -++ emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); -++} -+ -+-static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) -++static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) -+ { -+- _emit(cond, ARM_LDRB_I(ARM_R3, r_addr, 1), ctx); -+- _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); -+- _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 3), ctx); -+- _emit(cond, ARM_LSL_I(ARM_R3, ARM_R3, 16), ctx); -+- _emit(cond, ARM_LDRB_I(ARM_R0, r_addr, 2), ctx); -+- _emit(cond, ARM_ORR_S(ARM_R3, ARM_R3, ARM_R1, SRTYPE_LSL, 24), ctx); -+- _emit(cond, ARM_ORR_R(ARM_R3, ARM_R3, ARM_R2), ctx); -+- _emit(cond, ARM_ORR_S(r_res, ARM_R3, ARM_R0, SRTYPE_LSL, 8), ctx); -++#if __LINUX_ARM_ARCH__ < 5 -++ emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); -++ emit_bx_r(tgt_reg, ctx); -++#else -++ emit(ARM_BLX_R(tgt_reg), ctx); -++#endif -+ } -+ -+-static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) -++static inline int epilogue_offset(const struct jit_ctx *ctx) -+ { -+- _emit(cond, ARM_LDRB_I(ARM_R1, r_addr, 0), ctx); -+- _emit(cond, ARM_LDRB_I(ARM_R2, r_addr, 1), ctx); -+- _emit(cond, ARM_ORR_S(r_res, ARM_R2, ARM_R1, SRTYPE_LSL, 8), ctx); -++ int to, from; -++ /* No need for 1st dummy run */ -++ if (ctx->target == NULL) -++ return 0; -++ to = ctx->epilogue_offset; -++ from = ctx->idx; -++ -++ return to - from - 2; -+ } -+ -+-static inline void emit_swap16(u8 r_dst, u8 r_src, struct jit_ctx *ctx) -++static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op) -+ { -+- /* r_dst = (r_src << 8) | (r_src >> 8) */ -+- emit(ARM_LSL_I(ARM_R1, r_src, 8), ctx); -+- emit(ARM_ORR_S(r_dst, ARM_R1, r_src, SRTYPE_LSR, 8), ctx); -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ -++#if __LINUX_ARM_ARCH__ == 7 -++ if (elf_hwcap & HWCAP_IDIVA) { -++ if (op == BPF_DIV) -++ emit(ARM_UDIV(rd, rm, rn), ctx); -++ else { -++ emit(ARM_UDIV(ARM_IP, rm, rn), ctx); -++ emit(ARM_MLS(rd, rn, ARM_IP, rm), ctx); -++ } -++ return; -++ } -++#endif -+ -+ /* -+- * we need to mask out the bits set in r_dst[23:16] due to -+- * the first shift instruction. -+- * -+- * note that 0x8ff is the encoded immediate 0x00ff0000. -++ * For BPF_ALU | BPF_DIV | BPF_K instructions -++ * As ARM_R1 and ARM_R0 contains 1st argument of bpf -++ * function, we need to save it on caller side to save -++ * it from getting destroyed within callee. -++ * After the return from the callee, we restore ARM_R0 -++ * ARM_R1. -+ */ -+- emit(ARM_BIC_I(r_dst, r_dst, 0x8ff), ctx); -++ if (rn != ARM_R1) { -++ emit(ARM_MOV_R(tmp[0], ARM_R1), ctx); -++ emit(ARM_MOV_R(ARM_R1, rn), ctx); -++ } -++ if (rm != ARM_R0) { -++ emit(ARM_MOV_R(tmp[1], ARM_R0), ctx); -++ emit(ARM_MOV_R(ARM_R0, rm), ctx); -++ } -++ -++ /* Call appropriate function */ -++ emit_mov_i(ARM_IP, op == BPF_DIV ? -++ (u32)jit_udiv32 : (u32)jit_mod32, ctx); -++ emit_blx_r(ARM_IP, ctx); -++ -++ /* Save return value */ -++ if (rd != ARM_R0) -++ emit(ARM_MOV_R(rd, ARM_R0), ctx); -++ -++ /* Restore ARM_R0 and ARM_R1 */ -++ if (rn != ARM_R1) -++ emit(ARM_MOV_R(ARM_R1, tmp[0]), ctx); -++ if (rm != ARM_R0) -++ emit(ARM_MOV_R(ARM_R0, tmp[1]), ctx); -+ } -+ -+-#else /* ARMv6+ */ -++/* Is the translated BPF register on stack? */ -++static bool is_stacked(s8 reg) -++{ -++ return reg < 0; -++} -+ -+-static void emit_load_be32(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) -++/* If a BPF register is on the stack (stk is true), load it to the -++ * supplied temporary register and return the temporary register -++ * for subsequent operations, otherwise just use the CPU register. -++ */ -++static s8 arm_bpf_get_reg32(s8 reg, s8 tmp, struct jit_ctx *ctx) -+ { -+- _emit(cond, ARM_LDR_I(r_res, r_addr, 0), ctx); -+-#ifdef __LITTLE_ENDIAN -+- _emit(cond, ARM_REV(r_res, r_res), ctx); -+-#endif -++ if (is_stacked(reg)) { -++ emit(ARM_LDR_I(tmp, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); -++ reg = tmp; -++ } -++ return reg; -+ } -+ -+-static void emit_load_be16(u8 cond, u8 r_res, u8 r_addr, struct jit_ctx *ctx) -++static const s8 *arm_bpf_get_reg64(const s8 *reg, const s8 *tmp, -++ struct jit_ctx *ctx) -+ { -+- _emit(cond, ARM_LDRH_I(r_res, r_addr, 0), ctx); -+-#ifdef __LITTLE_ENDIAN -+- _emit(cond, ARM_REV16(r_res, r_res), ctx); -+-#endif -++ if (is_stacked(reg[1])) { -++ if (__LINUX_ARM_ARCH__ >= 6 || -++ ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { -++ emit(ARM_LDRD_I(tmp[1], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); -++ } else { -++ emit(ARM_LDR_I(tmp[1], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); -++ emit(ARM_LDR_I(tmp[0], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); -++ } -++ reg = tmp; -++ } -++ return reg; -+ } -+ -+-static inline void emit_swap16(u8 r_dst __maybe_unused, -+- u8 r_src __maybe_unused, -+- struct jit_ctx *ctx __maybe_unused) -++/* If a BPF register is on the stack (stk is true), save the register -++ * back to the stack. If the source register is not the same, then -++ * move it into the correct register. -++ */ -++static void arm_bpf_put_reg32(s8 reg, s8 src, struct jit_ctx *ctx) -+ { -+-#ifdef __LITTLE_ENDIAN -+- emit(ARM_REV16(r_dst, r_src), ctx); -+-#endif -++ if (is_stacked(reg)) -++ emit(ARM_STR_I(src, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(reg)), ctx); -++ else if (reg != src) -++ emit(ARM_MOV_R(reg, src), ctx); -++} -++ -++static void arm_bpf_put_reg64(const s8 *reg, const s8 *src, -++ struct jit_ctx *ctx) -++{ -++ if (is_stacked(reg[1])) { -++ if (__LINUX_ARM_ARCH__ >= 6 || -++ ctx->cpu_architecture >= CPU_ARCH_ARMv5TE) { -++ emit(ARM_STRD_I(src[1], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); -++ } else { -++ emit(ARM_STR_I(src[1], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[1])), ctx); -++ emit(ARM_STR_I(src[0], ARM_FP, -++ EBPF_SCRATCH_TO_ARM_FP(reg[0])), ctx); -++ } -++ } else { -++ if (reg[1] != src[1]) -++ emit(ARM_MOV_R(reg[1], src[1]), ctx); -++ if (reg[0] != src[0]) -++ emit(ARM_MOV_R(reg[0], src[0]), ctx); -++ } -+ } -+ -+-#endif /* __LINUX_ARM_ARCH__ < 6 */ -++static inline void emit_a32_mov_i(const s8 dst, const u32 val, -++ struct jit_ctx *ctx) -++{ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -+ -++ if (is_stacked(dst)) { -++ emit_mov_i(tmp[1], val, ctx); -++ arm_bpf_put_reg32(dst, tmp[1], ctx); -++ } else { -++ emit_mov_i(dst, val, ctx); -++ } -++} -+ -+-/* Compute the immediate value for a PC-relative branch. */ -+-static inline u32 b_imm(unsigned tgt, struct jit_ctx *ctx) -++static void emit_a32_mov_i64(const s8 dst[], u64 val, struct jit_ctx *ctx) -+ { -+- u32 imm; -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *rd = is_stacked(dst_lo) ? tmp : dst; -+ -+- if (ctx->target == NULL) -+- return 0; -+- /* -+- * BPF allows only forward jumps and the offset of the target is -+- * still the one computed during the first pass. -++ emit_mov_i(rd[1], (u32)val, ctx); -++ emit_mov_i(rd[0], val >> 32, ctx); -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++/* Sign extended move */ -++static inline void emit_a32_mov_se_i64(const bool is64, const s8 dst[], -++ const u32 val, struct jit_ctx *ctx) { -++ u64 val64 = val; -++ -++ if (is64 && (val & (1<<31))) -++ val64 |= 0xffffffff00000000ULL; -++ emit_a32_mov_i64(dst, val64, ctx); -++} -++ -++static inline void emit_a32_add_r(const u8 dst, const u8 src, -++ const bool is64, const bool hi, -++ struct jit_ctx *ctx) { -++ /* 64 bit : -++ * adds dst_lo, dst_lo, src_lo -++ * adc dst_hi, dst_hi, src_hi -++ * 32 bit : -++ * add dst_lo, dst_lo, src_lo -+ */ -+- imm = ctx->offsets[tgt] + ctx->prologue_bytes - (ctx->idx * 4 + 8); -++ if (!hi && is64) -++ emit(ARM_ADDS_R(dst, dst, src), ctx); -++ else if (hi && is64) -++ emit(ARM_ADC_R(dst, dst, src), ctx); -++ else -++ emit(ARM_ADD_R(dst, dst, src), ctx); -++} -+ -+- return imm >> 2; -++static inline void emit_a32_sub_r(const u8 dst, const u8 src, -++ const bool is64, const bool hi, -++ struct jit_ctx *ctx) { -++ /* 64 bit : -++ * subs dst_lo, dst_lo, src_lo -++ * sbc dst_hi, dst_hi, src_hi -++ * 32 bit : -++ * sub dst_lo, dst_lo, src_lo -++ */ -++ if (!hi && is64) -++ emit(ARM_SUBS_R(dst, dst, src), ctx); -++ else if (hi && is64) -++ emit(ARM_SBC_R(dst, dst, src), ctx); -++ else -++ emit(ARM_SUB_R(dst, dst, src), ctx); -++} -++ -++static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, -++ const bool hi, const u8 op, struct jit_ctx *ctx){ -++ switch (BPF_OP(op)) { -++ /* dst = dst + src */ -++ case BPF_ADD: -++ emit_a32_add_r(dst, src, is64, hi, ctx); -++ break; -++ /* dst = dst - src */ -++ case BPF_SUB: -++ emit_a32_sub_r(dst, src, is64, hi, ctx); -++ break; -++ /* dst = dst | src */ -++ case BPF_OR: -++ emit(ARM_ORR_R(dst, dst, src), ctx); -++ break; -++ /* dst = dst & src */ -++ case BPF_AND: -++ emit(ARM_AND_R(dst, dst, src), ctx); -++ break; -++ /* dst = dst ^ src */ -++ case BPF_XOR: -++ emit(ARM_EOR_R(dst, dst, src), ctx); -++ break; -++ /* dst = dst * src */ -++ case BPF_MUL: -++ emit(ARM_MUL(dst, dst, src), ctx); -++ break; -++ /* dst = dst << src */ -++ case BPF_LSH: -++ emit(ARM_LSL_R(dst, dst, src), ctx); -++ break; -++ /* dst = dst >> src */ -++ case BPF_RSH: -++ emit(ARM_LSR_R(dst, dst, src), ctx); -++ break; -++ /* dst = dst >> src (signed)*/ -++ case BPF_ARSH: -++ emit(ARM_MOV_SR(dst, dst, SRTYPE_ASR, src), ctx); -++ break; -++ } -+ } -+ -+-#define OP_IMM3(op, r1, r2, imm_val, ctx) \ -+- do { \ -+- imm12 = imm8m(imm_val); \ -+- if (imm12 < 0) { \ -+- emit_mov_i_no8m(r_scratch, imm_val, ctx); \ -+- emit(op ## _R((r1), (r2), r_scratch), ctx); \ -+- } else { \ -+- emit(op ## _I((r1), (r2), imm12), ctx); \ -+- } \ -+- } while (0) -+- -+-static inline void emit_err_ret(u8 cond, struct jit_ctx *ctx) -+-{ -+- if (ctx->ret0_fp_idx >= 0) { -+- _emit(cond, ARM_B(b_imm(ctx->ret0_fp_idx, ctx)), ctx); -+- /* NOP to keep the size constant between passes */ -+- emit(ARM_MOV_R(ARM_R0, ARM_R0), ctx); -++/* ALU operation (32 bit) -++ * dst = dst (op) src -++ */ -++static inline void emit_a32_alu_r(const s8 dst, const s8 src, -++ struct jit_ctx *ctx, const bool is64, -++ const bool hi, const u8 op) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ s8 rn, rd; -++ -++ rn = arm_bpf_get_reg32(src, tmp[1], ctx); -++ rd = arm_bpf_get_reg32(dst, tmp[0], ctx); -++ /* ALU operation */ -++ emit_alu_r(rd, rn, is64, hi, op, ctx); -++ arm_bpf_put_reg32(dst, rd, ctx); -++} -++ -++/* ALU operation (64 bit) */ -++static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], -++ const s8 src[], struct jit_ctx *ctx, -++ const u8 op) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ if (is64) { -++ const s8 *rs; -++ -++ rs = arm_bpf_get_reg64(src, tmp2, ctx); -++ -++ /* ALU operation */ -++ emit_alu_r(rd[1], rs[1], true, false, op, ctx); -++ emit_alu_r(rd[0], rs[0], true, true, op, ctx); -+ } else { -+- _emit(cond, ARM_MOV_I(ARM_R0, 0), ctx); -+- _emit(cond, ARM_B(b_imm(ctx->skf->len, ctx)), ctx); -++ s8 rs; -++ -++ rs = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ -++ /* ALU operation */ -++ emit_alu_r(rd[1], rs, true, false, op, ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(rd[0], 0, ctx); -+ } -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -+ } -+ -+-static inline void emit_blx_r(u8 tgt_reg, struct jit_ctx *ctx) -+-{ -+-#if __LINUX_ARM_ARCH__ < 5 -+- emit(ARM_MOV_R(ARM_LR, ARM_PC), ctx); -++/* dst = src (4 bytes)*/ -++static inline void emit_a32_mov_r(const s8 dst, const s8 src, -++ struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ s8 rt; -++ -++ rt = arm_bpf_get_reg32(src, tmp[0], ctx); -++ arm_bpf_put_reg32(dst, rt, ctx); -++} -++ -++/* dst = src */ -++static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], -++ const s8 src[], -++ struct jit_ctx *ctx) { -++ if (!is64) { -++ emit_a32_mov_r(dst_lo, src_lo, ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ /* Zero out high 4 bytes */ -++ emit_a32_mov_i(dst_hi, 0, ctx); -++ } else if (__LINUX_ARM_ARCH__ < 6 && -++ ctx->cpu_architecture < CPU_ARCH_ARMv5TE) { -++ /* complete 8 byte move */ -++ emit_a32_mov_r(dst_lo, src_lo, ctx); -++ emit_a32_mov_r(dst_hi, src_hi, ctx); -++ } else if (is_stacked(src_lo) && is_stacked(dst_lo)) { -++ const u8 *tmp = bpf2a32[TMP_REG_1]; -++ -++ emit(ARM_LDRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); -++ emit(ARM_STRD_I(tmp[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); -++ } else if (is_stacked(src_lo)) { -++ emit(ARM_LDRD_I(dst[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(src_lo)), ctx); -++ } else if (is_stacked(dst_lo)) { -++ emit(ARM_STRD_I(src[1], ARM_FP, EBPF_SCRATCH_TO_ARM_FP(dst_lo)), ctx); -++ } else { -++ emit(ARM_MOV_R(dst[0], src[0]), ctx); -++ emit(ARM_MOV_R(dst[1], src[1]), ctx); -++ } -++} -+ -+- if (elf_hwcap & HWCAP_THUMB) -+- emit(ARM_BX(tgt_reg), ctx); -+- else -+- emit(ARM_MOV_R(ARM_PC, tgt_reg), ctx); -+-#else -+- emit(ARM_BLX_R(tgt_reg), ctx); -+-#endif -++/* Shift operations */ -++static inline void emit_a32_alu_i(const s8 dst, const u32 val, -++ struct jit_ctx *ctx, const u8 op) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ s8 rd; -++ -++ rd = arm_bpf_get_reg32(dst, tmp[0], ctx); -++ -++ /* Do shift operation */ -++ switch (op) { -++ case BPF_LSH: -++ emit(ARM_LSL_I(rd, rd, val), ctx); -++ break; -++ case BPF_RSH: -++ emit(ARM_LSR_I(rd, rd, val), ctx); -++ break; -++ case BPF_NEG: -++ emit(ARM_RSB_I(rd, rd, val), ctx); -++ break; -++ } -++ -++ arm_bpf_put_reg32(dst, rd, ctx); -+ } -+ -+-static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, -+- int bpf_op) -+-{ -+-#if __LINUX_ARM_ARCH__ == 7 -+- if (elf_hwcap & HWCAP_IDIVA) { -+- if (bpf_op == BPF_DIV) -+- emit(ARM_UDIV(rd, rm, rn), ctx); -+- else { -+- emit(ARM_UDIV(ARM_R3, rm, rn), ctx); -+- emit(ARM_MLS(rd, rn, ARM_R3, rm), ctx); -++/* dst = ~dst (64 bit) */ -++static inline void emit_a32_neg64(const s8 dst[], -++ struct jit_ctx *ctx){ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *rd; -++ -++ /* Setup Operand */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do Negate Operation */ -++ emit(ARM_RSBS_I(rd[1], rd[1], 0), ctx); -++ emit(ARM_RSC_I(rd[0], rd[0], 0), ctx); -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++/* dst = dst << src */ -++static inline void emit_a32_lsh_r64(const s8 dst[], const s8 src[], -++ struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ s8 rt; -++ -++ /* Setup Operands */ -++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do LSH operation */ -++ emit(ARM_SUB_I(ARM_IP, rt, 32), ctx); -++ emit(ARM_RSB_I(tmp2[0], rt, 32), ctx); -++ emit(ARM_MOV_SR(ARM_LR, rd[0], SRTYPE_ASL, rt), ctx); -++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[1], SRTYPE_ASL, ARM_IP), ctx); -++ emit(ARM_ORR_SR(ARM_IP, ARM_LR, rd[1], SRTYPE_LSR, tmp2[0]), ctx); -++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_ASL, rt), ctx); -++ -++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); -++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); -++} -++ -++/* dst = dst >> src (signed)*/ -++static inline void emit_a32_arsh_r64(const s8 dst[], const s8 src[], -++ struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ s8 rt; -++ -++ /* Setup Operands */ -++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do the ARSH operation */ -++ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); -++ emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); -++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); -++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); -++ _emit(ARM_COND_MI, ARM_B(0), ctx); -++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASR, tmp2[0]), ctx); -++ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_ASR, rt), ctx); -++ -++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); -++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); -++} -++ -++/* dst = dst >> src */ -++static inline void emit_a32_rsh_r64(const s8 dst[], const s8 src[], -++ struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ s8 rt; -++ -++ /* Setup Operands */ -++ rt = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do RSH operation */ -++ emit(ARM_RSB_I(ARM_IP, rt, 32), ctx); -++ emit(ARM_SUBS_I(tmp2[0], rt, 32), ctx); -++ emit(ARM_MOV_SR(ARM_LR, rd[1], SRTYPE_LSR, rt), ctx); -++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_ASL, ARM_IP), ctx); -++ emit(ARM_ORR_SR(ARM_LR, ARM_LR, rd[0], SRTYPE_LSR, tmp2[0]), ctx); -++ emit(ARM_MOV_SR(ARM_IP, rd[0], SRTYPE_LSR, rt), ctx); -++ -++ arm_bpf_put_reg32(dst_lo, ARM_LR, ctx); -++ arm_bpf_put_reg32(dst_hi, ARM_IP, ctx); -++} -++ -++/* dst = dst << val */ -++static inline void emit_a32_lsh_i64(const s8 dst[], -++ const u32 val, struct jit_ctx *ctx){ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ -++ /* Setup operands */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do LSH operation */ -++ if (val < 32) { -++ emit(ARM_MOV_SI(tmp2[0], rd[0], SRTYPE_ASL, val), ctx); -++ emit(ARM_ORR_SI(rd[0], tmp2[0], rd[1], SRTYPE_LSR, 32 - val), ctx); -++ emit(ARM_MOV_SI(rd[1], rd[1], SRTYPE_ASL, val), ctx); -++ } else { -++ if (val == 32) -++ emit(ARM_MOV_R(rd[0], rd[1]), ctx); -++ else -++ emit(ARM_MOV_SI(rd[0], rd[1], SRTYPE_ASL, val - 32), ctx); -++ emit(ARM_EOR_R(rd[1], rd[1], rd[1]), ctx); -++ } -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++/* dst = dst >> val */ -++static inline void emit_a32_rsh_i64(const s8 dst[], -++ const u32 val, struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ -++ /* Setup operands */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do LSR operation */ -++ if (val == 0) { -++ /* An immediate value of 0 encodes a shift amount of 32 -++ * for LSR. To shift by 0, don't do anything. -++ */ -++ } else if (val < 32) { -++ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); -++ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); -++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_LSR, val), ctx); -++ } else if (val == 32) { -++ emit(ARM_MOV_R(rd[1], rd[0]), ctx); -++ emit(ARM_MOV_I(rd[0], 0), ctx); -++ } else { -++ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_LSR, val - 32), ctx); -++ emit(ARM_MOV_I(rd[0], 0), ctx); -++ } -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++/* dst = dst >> val (signed) */ -++static inline void emit_a32_arsh_i64(const s8 dst[], -++ const u32 val, struct jit_ctx *ctx){ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd; -++ -++ /* Setup operands */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Do ARSH operation */ -++ if (val == 0) { -++ /* An immediate value of 0 encodes a shift amount of 32 -++ * for ASR. To shift by 0, don't do anything. -++ */ -++ } else if (val < 32) { -++ emit(ARM_MOV_SI(tmp2[1], rd[1], SRTYPE_LSR, val), ctx); -++ emit(ARM_ORR_SI(rd[1], tmp2[1], rd[0], SRTYPE_ASL, 32 - val), ctx); -++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, val), ctx); -++ } else if (val == 32) { -++ emit(ARM_MOV_R(rd[1], rd[0]), ctx); -++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); -++ } else { -++ emit(ARM_MOV_SI(rd[1], rd[0], SRTYPE_ASR, val - 32), ctx); -++ emit(ARM_MOV_SI(rd[0], rd[0], SRTYPE_ASR, 31), ctx); -++ } -++ -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++static inline void emit_a32_mul_r64(const s8 dst[], const s8 src[], -++ struct jit_ctx *ctx) { -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rd, *rt; -++ -++ /* Setup operands for multiplication */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ rt = arm_bpf_get_reg64(src, tmp2, ctx); -++ -++ /* Do Multiplication */ -++ emit(ARM_MUL(ARM_IP, rd[1], rt[0]), ctx); -++ emit(ARM_MUL(ARM_LR, rd[0], rt[1]), ctx); -++ emit(ARM_ADD_R(ARM_LR, ARM_IP, ARM_LR), ctx); -++ -++ emit(ARM_UMULL(ARM_IP, rd[0], rd[1], rt[1]), ctx); -++ emit(ARM_ADD_R(rd[0], ARM_LR, rd[0]), ctx); -++ -++ arm_bpf_put_reg32(dst_lo, ARM_IP, ctx); -++ arm_bpf_put_reg32(dst_hi, rd[0], ctx); -++} -++ -++static bool is_ldst_imm(s16 off, const u8 size) -++{ -++ s16 off_max = 0; -++ -++ switch (size) { -++ case BPF_B: -++ case BPF_W: -++ off_max = 0xfff; -++ break; -++ case BPF_H: -++ off_max = 0xff; -++ break; -++ case BPF_DW: -++ /* Need to make sure off+4 does not overflow. */ -++ off_max = 0xfff - 4; -++ break; -++ } -++ return -off_max <= off && off <= off_max; -++} -++ -++/* *(size *)(dst + off) = src */ -++static inline void emit_str_r(const s8 dst, const s8 src[], -++ s16 off, struct jit_ctx *ctx, const u8 sz){ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ s8 rd; -++ -++ rd = arm_bpf_get_reg32(dst, tmp[1], ctx); -++ -++ if (!is_ldst_imm(off, sz)) { -++ emit_a32_mov_i(tmp[0], off, ctx); -++ emit(ARM_ADD_R(tmp[0], tmp[0], rd), ctx); -++ rd = tmp[0]; -++ off = 0; -++ } -++ switch (sz) { -++ case BPF_B: -++ /* Store a Byte */ -++ emit(ARM_STRB_I(src_lo, rd, off), ctx); -++ break; -++ case BPF_H: -++ /* Store a HalfWord */ -++ emit(ARM_STRH_I(src_lo, rd, off), ctx); -++ break; -++ case BPF_W: -++ /* Store a Word */ -++ emit(ARM_STR_I(src_lo, rd, off), ctx); -++ break; -++ case BPF_DW: -++ /* Store a Double Word */ -++ emit(ARM_STR_I(src_lo, rd, off), ctx); -++ emit(ARM_STR_I(src_hi, rd, off + 4), ctx); -++ break; -++ } -++} -++ -++/* dst = *(size*)(src + off) */ -++static inline void emit_ldx_r(const s8 dst[], const s8 src, -++ s16 off, struct jit_ctx *ctx, const u8 sz){ -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *rd = is_stacked(dst_lo) ? tmp : dst; -++ s8 rm = src; -++ -++ if (!is_ldst_imm(off, sz)) { -++ emit_a32_mov_i(tmp[0], off, ctx); -++ emit(ARM_ADD_R(tmp[0], tmp[0], src), ctx); -++ rm = tmp[0]; -++ off = 0; -++ } else if (rd[1] == rm) { -++ emit(ARM_MOV_R(tmp[0], rm), ctx); -++ rm = tmp[0]; -++ } -++ switch (sz) { -++ case BPF_B: -++ /* Load a Byte */ -++ emit(ARM_LDRB_I(rd[1], rm, off), ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(rd[0], 0, ctx); -++ break; -++ case BPF_H: -++ /* Load a HalfWord */ -++ emit(ARM_LDRH_I(rd[1], rm, off), ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(rd[0], 0, ctx); -++ break; -++ case BPF_W: -++ /* Load a Word */ -++ emit(ARM_LDR_I(rd[1], rm, off), ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(rd[0], 0, ctx); -++ break; -++ case BPF_DW: -++ /* Load a Double Word */ -++ emit(ARM_LDR_I(rd[1], rm, off), ctx); -++ emit(ARM_LDR_I(rd[0], rm, off + 4), ctx); -++ break; -++ } -++ arm_bpf_put_reg64(dst, rd, ctx); -++} -++ -++/* Arithmatic Operation */ -++static inline void emit_ar_r(const u8 rd, const u8 rt, const u8 rm, -++ const u8 rn, struct jit_ctx *ctx, u8 op, -++ bool is_jmp64) { -++ switch (op) { -++ case BPF_JSET: -++ if (is_jmp64) { -++ emit(ARM_AND_R(ARM_IP, rt, rn), ctx); -++ emit(ARM_AND_R(ARM_LR, rd, rm), ctx); -++ emit(ARM_ORRS_R(ARM_IP, ARM_LR, ARM_IP), ctx); -++ } else { -++ emit(ARM_ANDS_R(ARM_IP, rt, rn), ctx); -+ } -+- return; -++ break; -++ case BPF_JEQ: -++ case BPF_JNE: -++ case BPF_JGT: -++ case BPF_JGE: -++ case BPF_JLE: -++ case BPF_JLT: -++ if (is_jmp64) { -++ emit(ARM_CMP_R(rd, rm), ctx); -++ /* Only compare low halve if high halve are equal. */ -++ _emit(ARM_COND_EQ, ARM_CMP_R(rt, rn), ctx); -++ } else { -++ emit(ARM_CMP_R(rt, rn), ctx); -++ } -++ break; -++ case BPF_JSLE: -++ case BPF_JSGT: -++ emit(ARM_CMP_R(rn, rt), ctx); -++ if (is_jmp64) -++ emit(ARM_SBCS_R(ARM_IP, rm, rd), ctx); -++ break; -++ case BPF_JSLT: -++ case BPF_JSGE: -++ emit(ARM_CMP_R(rt, rn), ctx); -++ if (is_jmp64) -++ emit(ARM_SBCS_R(ARM_IP, rd, rm), ctx); -++ break; -+ } -+-#endif -++} -+ -+- /* -+- * For BPF_ALU | BPF_DIV | BPF_K instructions, rm is ARM_R4 -+- * (r_A) and rn is ARM_R0 (r_scratch) so load rn first into -+- * ARM_R1 to avoid accidentally overwriting ARM_R0 with rm -+- * before using it as a source for ARM_R1. -+- * -+- * For BPF_ALU | BPF_DIV | BPF_X rm is ARM_R4 (r_A) and rn is -+- * ARM_R5 (r_X) so there is no particular register overlap -+- * issues. -++static int out_offset = -1; /* initialized on the first pass of build_body() */ -++static int emit_bpf_tail_call(struct jit_ctx *ctx) -++{ -++ -++ /* bpf_tail_call(void *prog_ctx, struct bpf_array *array, u64 index) */ -++ const s8 *r2 = bpf2a32[BPF_REG_2]; -++ const s8 *r3 = bpf2a32[BPF_REG_3]; -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *tcc = bpf2a32[TCALL_CNT]; -++ const s8 *tc; -++ const int idx0 = ctx->idx; -++#define cur_offset (ctx->idx - idx0) -++#define jmp_offset (out_offset - (cur_offset) - 2) -++ u32 lo, hi; -++ s8 r_array, r_index; -++ int off; -++ -++ /* if (index >= array->map.max_entries) -++ * goto out; -+ */ -+- if (rn != ARM_R1) -+- emit(ARM_MOV_R(ARM_R1, rn), ctx); -+- if (rm != ARM_R0) -+- emit(ARM_MOV_R(ARM_R0, rm), ctx); -++ BUILD_BUG_ON(offsetof(struct bpf_array, map.max_entries) > -++ ARM_INST_LDST__IMM12); -++ off = offsetof(struct bpf_array, map.max_entries); -++ r_array = arm_bpf_get_reg32(r2[1], tmp2[0], ctx); -++ /* index is 32-bit for arrays */ -++ r_index = arm_bpf_get_reg32(r3[1], tmp2[1], ctx); -++ /* array->map.max_entries */ -++ emit(ARM_LDR_I(tmp[1], r_array, off), ctx); -++ /* index >= array->map.max_entries */ -++ emit(ARM_CMP_R(r_index, tmp[1]), ctx); -++ _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); -++ -++ /* tmp2[0] = array, tmp2[1] = index */ -++ -++ /* if (tail_call_cnt > MAX_TAIL_CALL_CNT) -++ * goto out; -++ * tail_call_cnt++; -++ */ -++ lo = (u32)MAX_TAIL_CALL_CNT; -++ hi = (u32)((u64)MAX_TAIL_CALL_CNT >> 32); -++ tc = arm_bpf_get_reg64(tcc, tmp, ctx); -++ emit(ARM_CMP_I(tc[0], hi), ctx); -++ _emit(ARM_COND_EQ, ARM_CMP_I(tc[1], lo), ctx); -++ _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); -++ emit(ARM_ADDS_I(tc[1], tc[1], 1), ctx); -++ emit(ARM_ADC_I(tc[0], tc[0], 0), ctx); -++ arm_bpf_put_reg64(tcc, tmp, ctx); -++ -++ /* prog = array->ptrs[index] -++ * if (prog == NULL) -++ * goto out; -++ */ -++ BUILD_BUG_ON(imm8m(offsetof(struct bpf_array, ptrs)) < 0); -++ off = imm8m(offsetof(struct bpf_array, ptrs)); -++ emit(ARM_ADD_I(tmp[1], r_array, off), ctx); -++ emit(ARM_LDR_R_SI(tmp[1], tmp[1], r_index, SRTYPE_ASL, 2), ctx); -++ emit(ARM_CMP_I(tmp[1], 0), ctx); -++ _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); -++ -++ /* goto *(prog->bpf_func + prologue_size); */ -++ BUILD_BUG_ON(offsetof(struct bpf_prog, bpf_func) > -++ ARM_INST_LDST__IMM12); -++ off = offsetof(struct bpf_prog, bpf_func); -++ emit(ARM_LDR_I(tmp[1], tmp[1], off), ctx); -++ emit(ARM_ADD_I(tmp[1], tmp[1], ctx->prologue_bytes), ctx); -++ emit_bx_r(tmp[1], ctx); -++ -++ /* out: */ -++ if (out_offset == -1) -++ out_offset = cur_offset; -++ if (cur_offset != out_offset) { -++ pr_err_once("tail_call out_offset = %d, expected %d!\n", -++ cur_offset, out_offset); -++ return -1; -++ } -++ return 0; -++#undef cur_offset -++#undef jmp_offset -++} -++ -++/* 0xabcd => 0xcdab */ -++static inline void emit_rev16(const u8 rd, const u8 rn, struct jit_ctx *ctx) -++{ -++#if __LINUX_ARM_ARCH__ < 6 -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -+ -+- ctx->seen |= SEEN_CALL; -+- emit_mov_i(ARM_R3, bpf_op == BPF_DIV ? (u32)jit_udiv : (u32)jit_mod, -+- ctx); -+- emit_blx_r(ARM_R3, ctx); -++ emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); -++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 8), ctx); -++ emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); -++ emit(ARM_ORR_SI(rd, tmp2[0], tmp2[1], SRTYPE_LSL, 8), ctx); -++#else /* ARMv6+ */ -++ emit(ARM_REV16(rd, rn), ctx); -++#endif -++} -+ -+- if (rd != ARM_R0) -+- emit(ARM_MOV_R(rd, ARM_R0), ctx); -++/* 0xabcdefgh => 0xghefcdab */ -++static inline void emit_rev32(const u8 rd, const u8 rn, struct jit_ctx *ctx) -++{ -++#if __LINUX_ARM_ARCH__ < 6 -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ -++ emit(ARM_AND_I(tmp2[1], rn, 0xff), ctx); -++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 24), ctx); -++ emit(ARM_ORR_SI(ARM_IP, tmp2[0], tmp2[1], SRTYPE_LSL, 24), ctx); -++ -++ emit(ARM_MOV_SI(tmp2[1], rn, SRTYPE_LSR, 8), ctx); -++ emit(ARM_AND_I(tmp2[1], tmp2[1], 0xff), ctx); -++ emit(ARM_MOV_SI(tmp2[0], rn, SRTYPE_LSR, 16), ctx); -++ emit(ARM_AND_I(tmp2[0], tmp2[0], 0xff), ctx); -++ emit(ARM_MOV_SI(tmp2[0], tmp2[0], SRTYPE_LSL, 8), ctx); -++ emit(ARM_ORR_SI(tmp2[0], tmp2[0], tmp2[1], SRTYPE_LSL, 16), ctx); -++ emit(ARM_ORR_R(rd, ARM_IP, tmp2[0]), ctx); -++ -++#else /* ARMv6+ */ -++ emit(ARM_REV(rd, rn), ctx); -++#endif -+ } -+ -+-static inline void update_on_xread(struct jit_ctx *ctx) -++// push the scratch stack register on top of the stack -++static inline void emit_push_r64(const s8 src[], struct jit_ctx *ctx) -+ { -+- if (!(ctx->seen & SEEN_X)) -+- ctx->flags |= FLAG_NEED_X_RESET; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s8 *rt; -++ u16 reg_set = 0; -++ -++ rt = arm_bpf_get_reg64(src, tmp2, ctx); -+ -+- ctx->seen |= SEEN_X; -++ reg_set = (1 << rt[1]) | (1 << rt[0]); -++ emit(ARM_PUSH(reg_set), ctx); -+ } -+ -+-static int build_body(struct jit_ctx *ctx) -++static void build_prologue(struct jit_ctx *ctx) -+ { -+- void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w}; -+- const struct bpf_prog *prog = ctx->skf; -+- const struct sock_filter *inst; -+- unsigned i, load_order, off, condt; -+- int imm12; -+- u32 k; -++ const s8 r0 = bpf2a32[BPF_REG_0][1]; -++ const s8 r2 = bpf2a32[BPF_REG_1][1]; -++ const s8 r3 = bpf2a32[BPF_REG_1][0]; -++ const s8 r4 = bpf2a32[BPF_REG_6][1]; -++ const s8 fplo = bpf2a32[BPF_REG_FP][1]; -++ const s8 fphi = bpf2a32[BPF_REG_FP][0]; -++ const s8 *tcc = bpf2a32[TCALL_CNT]; -+ -+- for (i = 0; i < prog->len; i++) { -+- u16 code; -++ /* Save callee saved registers. */ -++#ifdef CONFIG_FRAME_POINTER -++ u16 reg_set = CALLEE_PUSH_MASK | 1 << ARM_IP | 1 << ARM_PC; -++ emit(ARM_MOV_R(ARM_IP, ARM_SP), ctx); -++ emit(ARM_PUSH(reg_set), ctx); -++ emit(ARM_SUB_I(ARM_FP, ARM_IP, 4), ctx); -++#else -++ emit(ARM_PUSH(CALLEE_PUSH_MASK), ctx); -++ emit(ARM_MOV_R(ARM_FP, ARM_SP), ctx); -++#endif -++ /* Save frame pointer for later */ -++ emit(ARM_SUB_I(ARM_IP, ARM_SP, SCRATCH_SIZE), ctx); -+ -+- inst = &(prog->insns[i]); -+- /* K as an immediate value operand */ -+- k = inst->k; -+- code = bpf_anc_helper(inst); -++ ctx->stack_size = imm8m(STACK_SIZE); -+ -+- /* compute offsets only in the fake pass */ -+- if (ctx->target == NULL) -+- ctx->offsets[i] = ctx->idx * 4; -++ /* Set up function call stack */ -++ emit(ARM_SUB_I(ARM_SP, ARM_SP, ctx->stack_size), ctx); -+ -+- switch (code) { -+- case BPF_LD | BPF_IMM: -+- emit_mov_i(r_A, k, ctx); -+- break; -+- case BPF_LD | BPF_W | BPF_LEN: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); -+- emit(ARM_LDR_I(r_A, r_skb, -+- offsetof(struct sk_buff, len)), ctx); -+- break; -+- case BPF_LD | BPF_MEM: -+- /* A = scratch[k] */ -+- ctx->seen |= SEEN_MEM_WORD(k); -+- emit(ARM_LDR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); -+- break; -+- case BPF_LD | BPF_W | BPF_ABS: -+- load_order = 2; -+- goto load; -+- case BPF_LD | BPF_H | BPF_ABS: -+- load_order = 1; -+- goto load; -+- case BPF_LD | BPF_B | BPF_ABS: -+- load_order = 0; -+-load: -+- emit_mov_i(r_off, k, ctx); -+-load_common: -+- ctx->seen |= SEEN_DATA | SEEN_CALL; -+- -+- if (load_order > 0) { -+- emit(ARM_SUB_I(r_scratch, r_skb_hl, -+- 1 << load_order), ctx); -+- emit(ARM_CMP_R(r_scratch, r_off), ctx); -+- condt = ARM_COND_GE; -+- } else { -+- emit(ARM_CMP_R(r_skb_hl, r_off), ctx); -+- condt = ARM_COND_HI; -+- } -++ /* Set up BPF prog stack base register */ -++ emit_a32_mov_r(fplo, ARM_IP, ctx); -++ emit_a32_mov_i(fphi, 0, ctx); -+ -+- /* -+- * test for negative offset, only if we are -+- * currently scheduled to take the fast -+- * path. this will update the flags so that -+- * the slowpath instruction are ignored if the -+- * offset is negative. -+- * -+- * for loard_order == 0 the HI condition will -+- * make loads at offset 0 take the slow path too. -+- */ -+- _emit(condt, ARM_CMP_I(r_off, 0), ctx); -++ /* mov r4, 0 */ -++ emit(ARM_MOV_I(r4, 0), ctx); -+ -+- _emit(condt, ARM_ADD_R(r_scratch, r_off, r_skb_data), -+- ctx); -++ /* Move BPF_CTX to BPF_R1 */ -++ emit(ARM_MOV_R(r3, r4), ctx); -++ emit(ARM_MOV_R(r2, r0), ctx); -++ /* Initialize Tail Count */ -++ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[0])), ctx); -++ emit(ARM_STR_I(r4, ARM_FP, EBPF_SCRATCH_TO_ARM_FP(tcc[1])), ctx); -++ /* end of prologue */ -++} -+ -+- if (load_order == 0) -+- _emit(condt, ARM_LDRB_I(r_A, r_scratch, 0), -+- ctx); -+- else if (load_order == 1) -+- emit_load_be16(condt, r_A, r_scratch, ctx); -+- else if (load_order == 2) -+- emit_load_be32(condt, r_A, r_scratch, ctx); -+- -+- _emit(condt, ARM_B(b_imm(i + 1, ctx)), ctx); -+- -+- /* the slowpath */ -+- emit_mov_i(ARM_R3, (u32)load_func[load_order], ctx); -+- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); -+- /* the offset is already in R1 */ -+- emit_blx_r(ARM_R3, ctx); -+- /* check the result of skb_copy_bits */ -+- emit(ARM_CMP_I(ARM_R1, 0), ctx); -+- emit_err_ret(ARM_COND_NE, ctx); -+- emit(ARM_MOV_R(r_A, ARM_R0), ctx); -+- break; -+- case BPF_LD | BPF_W | BPF_IND: -+- load_order = 2; -+- goto load_ind; -+- case BPF_LD | BPF_H | BPF_IND: -+- load_order = 1; -+- goto load_ind; -+- case BPF_LD | BPF_B | BPF_IND: -+- load_order = 0; -+-load_ind: -+- update_on_xread(ctx); -+- OP_IMM3(ARM_ADD, r_off, r_X, k, ctx); -+- goto load_common; -+- case BPF_LDX | BPF_IMM: -+- ctx->seen |= SEEN_X; -+- emit_mov_i(r_X, k, ctx); -+- break; -+- case BPF_LDX | BPF_W | BPF_LEN: -+- ctx->seen |= SEEN_X | SEEN_SKB; -+- emit(ARM_LDR_I(r_X, r_skb, -+- offsetof(struct sk_buff, len)), ctx); -+- break; -+- case BPF_LDX | BPF_MEM: -+- ctx->seen |= SEEN_X | SEEN_MEM_WORD(k); -+- emit(ARM_LDR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); -+- break; -+- case BPF_LDX | BPF_B | BPF_MSH: -+- /* x = ((*(frame + k)) & 0xf) << 2; */ -+- ctx->seen |= SEEN_X | SEEN_DATA | SEEN_CALL; -+- /* the interpreter should deal with the negative K */ -+- if ((int)k < 0) -+- return -1; -+- /* offset in r1: we might have to take the slow path */ -+- emit_mov_i(r_off, k, ctx); -+- emit(ARM_CMP_R(r_skb_hl, r_off), ctx); -+- -+- /* load in r0: common with the slowpath */ -+- _emit(ARM_COND_HI, ARM_LDRB_R(ARM_R0, r_skb_data, -+- ARM_R1), ctx); -+- /* -+- * emit_mov_i() might generate one or two instructions, -+- * the same holds for emit_blx_r() -+- */ -+- _emit(ARM_COND_HI, ARM_B(b_imm(i + 1, ctx) - 2), ctx); -++/* restore callee saved registers. */ -++static void build_epilogue(struct jit_ctx *ctx) -++{ -++#ifdef CONFIG_FRAME_POINTER -++ /* When using frame pointers, some additional registers need to -++ * be loaded. */ -++ u16 reg_set = CALLEE_POP_MASK | 1 << ARM_SP; -++ emit(ARM_SUB_I(ARM_SP, ARM_FP, hweight16(reg_set) * 4), ctx); -++ emit(ARM_LDM(ARM_SP, reg_set), ctx); -++#else -++ /* Restore callee saved registers. */ -++ emit(ARM_MOV_R(ARM_SP, ARM_FP), ctx); -++ emit(ARM_POP(CALLEE_POP_MASK), ctx); -++#endif -++} -+ -+- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); -+- /* r_off is r1 */ -+- emit_mov_i(ARM_R3, (u32)jit_get_skb_b, ctx); -+- emit_blx_r(ARM_R3, ctx); -+- /* check the return value of skb_copy_bits */ -+- emit(ARM_CMP_I(ARM_R1, 0), ctx); -+- emit_err_ret(ARM_COND_NE, ctx); -+- -+- emit(ARM_AND_I(r_X, ARM_R0, 0x00f), ctx); -+- emit(ARM_LSL_I(r_X, r_X, 2), ctx); -+- break; -+- case BPF_ST: -+- ctx->seen |= SEEN_MEM_WORD(k); -+- emit(ARM_STR_I(r_A, ARM_SP, SCRATCH_OFF(k)), ctx); -+- break; -+- case BPF_STX: -+- update_on_xread(ctx); -+- ctx->seen |= SEEN_MEM_WORD(k); -+- emit(ARM_STR_I(r_X, ARM_SP, SCRATCH_OFF(k)), ctx); -+- break; -+- case BPF_ALU | BPF_ADD | BPF_K: -+- /* A += K */ -+- OP_IMM3(ARM_ADD, r_A, r_A, k, ctx); -+- break; -+- case BPF_ALU | BPF_ADD | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_ADD_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_SUB | BPF_K: -+- /* A -= K */ -+- OP_IMM3(ARM_SUB, r_A, r_A, k, ctx); -+- break; -+- case BPF_ALU | BPF_SUB | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_SUB_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_MUL | BPF_K: -+- /* A *= K */ -+- emit_mov_i(r_scratch, k, ctx); -+- emit(ARM_MUL(r_A, r_A, r_scratch), ctx); -+- break; -+- case BPF_ALU | BPF_MUL | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_MUL(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_DIV | BPF_K: -+- if (k == 1) -+- break; -+- emit_mov_i(r_scratch, k, ctx); -+- emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_DIV); -+- break; -+- case BPF_ALU | BPF_DIV | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_CMP_I(r_X, 0), ctx); -+- emit_err_ret(ARM_COND_EQ, ctx); -+- emit_udivmod(r_A, r_A, r_X, ctx, BPF_DIV); -+- break; -+- case BPF_ALU | BPF_MOD | BPF_K: -+- if (k == 1) { -+- emit_mov_i(r_A, 0, ctx); -++/* -++ * Convert an eBPF instruction to native instruction, i.e -++ * JITs an eBPF instruction. -++ * Returns : -++ * 0 - Successfully JITed an 8-byte eBPF instruction -++ * >0 - Successfully JITed a 16-byte eBPF instruction -++ * <0 - Failed to JIT. -++ */ -++static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) -++{ -++ const u8 code = insn->code; -++ const s8 *dst = bpf2a32[insn->dst_reg]; -++ const s8 *src = bpf2a32[insn->src_reg]; -++ const s8 *tmp = bpf2a32[TMP_REG_1]; -++ const s8 *tmp2 = bpf2a32[TMP_REG_2]; -++ const s16 off = insn->off; -++ const s32 imm = insn->imm; -++ const int i = insn - ctx->prog->insnsi; -++ const bool is64 = BPF_CLASS(code) == BPF_ALU64; -++ const s8 *rd, *rs; -++ s8 rd_lo, rt, rm, rn; -++ s32 jmp_offset; -++ -++#define check_imm(bits, imm) do { \ -++ if ((imm) >= (1 << ((bits) - 1)) || \ -++ (imm) < -(1 << ((bits) - 1))) { \ -++ pr_info("[%2d] imm=%d(0x%x) out of range\n", \ -++ i, imm, imm); \ -++ return -EINVAL; \ -++ } \ -++} while (0) -++#define check_imm24(imm) check_imm(24, imm) -++ -++ switch (code) { -++ /* ALU operations */ -++ -++ /* dst = src */ -++ case BPF_ALU | BPF_MOV | BPF_K: -++ case BPF_ALU | BPF_MOV | BPF_X: -++ case BPF_ALU64 | BPF_MOV | BPF_K: -++ case BPF_ALU64 | BPF_MOV | BPF_X: -++ switch (BPF_SRC(code)) { -++ case BPF_X: -++ if (imm == 1) { -++ /* Special mov32 for zext */ -++ emit_a32_mov_i(dst_hi, 0, ctx); -+ break; -+ } -+- emit_mov_i(r_scratch, k, ctx); -+- emit_udivmod(r_A, r_A, r_scratch, ctx, BPF_MOD); -++ emit_a32_mov_r64(is64, dst, src, ctx); -+ break; -+- case BPF_ALU | BPF_MOD | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_CMP_I(r_X, 0), ctx); -+- emit_err_ret(ARM_COND_EQ, ctx); -+- emit_udivmod(r_A, r_A, r_X, ctx, BPF_MOD); -+- break; -+- case BPF_ALU | BPF_OR | BPF_K: -+- /* A |= K */ -+- OP_IMM3(ARM_ORR, r_A, r_A, k, ctx); -+- break; -+- case BPF_ALU | BPF_OR | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_ORR_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_XOR | BPF_K: -+- /* A ^= K; */ -+- OP_IMM3(ARM_EOR, r_A, r_A, k, ctx); -+- break; -+- case BPF_ANC | SKF_AD_ALU_XOR_X: -+- case BPF_ALU | BPF_XOR | BPF_X: -+- /* A ^= X */ -+- update_on_xread(ctx); -+- emit(ARM_EOR_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_AND | BPF_K: -+- /* A &= K */ -+- OP_IMM3(ARM_AND, r_A, r_A, k, ctx); -+- break; -+- case BPF_ALU | BPF_AND | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_AND_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_LSH | BPF_K: -+- if (unlikely(k > 31)) -+- return -1; -+- emit(ARM_LSL_I(r_A, r_A, k), ctx); -+- break; -+- case BPF_ALU | BPF_LSH | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_LSL_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_RSH | BPF_K: -+- if (unlikely(k > 31)) -+- return -1; -+- if (k) -+- emit(ARM_LSR_I(r_A, r_A, k), ctx); -+- break; -+- case BPF_ALU | BPF_RSH | BPF_X: -+- update_on_xread(ctx); -+- emit(ARM_LSR_R(r_A, r_A, r_X), ctx); -+- break; -+- case BPF_ALU | BPF_NEG: -+- /* A = -A */ -+- emit(ARM_RSB_I(r_A, r_A, 0), ctx); -+- break; -+- case BPF_JMP | BPF_JA: -+- /* pc += K */ -+- emit(ARM_B(b_imm(i + k + 1, ctx)), ctx); -+- break; -+- case BPF_JMP | BPF_JEQ | BPF_K: -+- /* pc += (A == K) ? pc->jt : pc->jf */ -+- condt = ARM_COND_EQ; -+- goto cmp_imm; -+- case BPF_JMP | BPF_JGT | BPF_K: -+- /* pc += (A > K) ? pc->jt : pc->jf */ -+- condt = ARM_COND_HI; -+- goto cmp_imm; -+- case BPF_JMP | BPF_JGE | BPF_K: -+- /* pc += (A >= K) ? pc->jt : pc->jf */ -+- condt = ARM_COND_HS; -+-cmp_imm: -+- imm12 = imm8m(k); -+- if (imm12 < 0) { -+- emit_mov_i_no8m(r_scratch, k, ctx); -+- emit(ARM_CMP_R(r_A, r_scratch), ctx); -+- } else { -+- emit(ARM_CMP_I(r_A, imm12), ctx); -+- } -+-cond_jump: -+- if (inst->jt) -+- _emit(condt, ARM_B(b_imm(i + inst->jt + 1, -+- ctx)), ctx); -+- if (inst->jf) -+- _emit(condt ^ 1, ARM_B(b_imm(i + inst->jf + 1, -+- ctx)), ctx); -+- break; -+- case BPF_JMP | BPF_JEQ | BPF_X: -+- /* pc += (A == X) ? pc->jt : pc->jf */ -+- condt = ARM_COND_EQ; -+- goto cmp_x; -+- case BPF_JMP | BPF_JGT | BPF_X: -+- /* pc += (A > X) ? pc->jt : pc->jf */ -+- condt = ARM_COND_HI; -+- goto cmp_x; -+- case BPF_JMP | BPF_JGE | BPF_X: -+- /* pc += (A >= X) ? pc->jt : pc->jf */ -+- condt = ARM_COND_CS; -+-cmp_x: -+- update_on_xread(ctx); -+- emit(ARM_CMP_R(r_A, r_X), ctx); -+- goto cond_jump; -+- case BPF_JMP | BPF_JSET | BPF_K: -+- /* pc += (A & K) ? pc->jt : pc->jf */ -+- condt = ARM_COND_NE; -+- /* not set iff all zeroes iff Z==1 iff EQ */ -+- -+- imm12 = imm8m(k); -+- if (imm12 < 0) { -+- emit_mov_i_no8m(r_scratch, k, ctx); -+- emit(ARM_TST_R(r_A, r_scratch), ctx); -+- } else { -+- emit(ARM_TST_I(r_A, imm12), ctx); -+- } -+- goto cond_jump; -+- case BPF_JMP | BPF_JSET | BPF_X: -+- /* pc += (A & X) ? pc->jt : pc->jf */ -+- update_on_xread(ctx); -+- condt = ARM_COND_NE; -+- emit(ARM_TST_R(r_A, r_X), ctx); -+- goto cond_jump; -+- case BPF_RET | BPF_A: -+- emit(ARM_MOV_R(ARM_R0, r_A), ctx); -+- goto b_epilogue; -+- case BPF_RET | BPF_K: -+- if ((k == 0) && (ctx->ret0_fp_idx < 0)) -+- ctx->ret0_fp_idx = i; -+- emit_mov_i(ARM_R0, k, ctx); -+-b_epilogue: -+- if (i != ctx->skf->len - 1) -+- emit(ARM_B(b_imm(prog->len, ctx)), ctx); -+- break; -+- case BPF_MISC | BPF_TAX: -+- /* X = A */ -+- ctx->seen |= SEEN_X; -+- emit(ARM_MOV_R(r_X, r_A), ctx); -+- break; -+- case BPF_MISC | BPF_TXA: -+- /* A = X */ -+- update_on_xread(ctx); -+- emit(ARM_MOV_R(r_A, r_X), ctx); -+- break; -+- case BPF_ANC | SKF_AD_PROTOCOL: -+- /* A = ntohs(skb->protocol) */ -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, -+- protocol) != 2); -+- off = offsetof(struct sk_buff, protocol); -+- emit(ARM_LDRH_I(r_scratch, r_skb, off), ctx); -+- emit_swap16(r_A, r_scratch, ctx); -+- break; -+- case BPF_ANC | SKF_AD_CPU: -+- /* r_scratch = current_thread_info() */ -+- OP_IMM3(ARM_BIC, r_scratch, ARM_SP, THREAD_SIZE - 1, ctx); -+- /* A = current_thread_info()->cpu */ -+- BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, cpu) != 4); -+- off = offsetof(struct thread_info, cpu); -+- emit(ARM_LDR_I(r_A, r_scratch, off), ctx); -+- break; -+- case BPF_ANC | SKF_AD_IFINDEX: -+- case BPF_ANC | SKF_AD_HATYPE: -+- /* A = skb->dev->ifindex */ -+- /* A = skb->dev->type */ -+- ctx->seen |= SEEN_SKB; -+- off = offsetof(struct sk_buff, dev); -+- emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); -+- -+- emit(ARM_CMP_I(r_scratch, 0), ctx); -+- emit_err_ret(ARM_COND_EQ, ctx); -+- -+- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, -+- ifindex) != 4); -+- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, -+- type) != 2); -+- -+- if (code == (BPF_ANC | SKF_AD_IFINDEX)) { -+- off = offsetof(struct net_device, ifindex); -+- emit(ARM_LDR_I(r_A, r_scratch, off), ctx); -+- } else { -+- /* -+- * offset of field "type" in "struct -+- * net_device" is above what can be -+- * used in the ldrh rd, [rn, #imm] -+- * instruction, so load the offset in -+- * a register and use ldrh rd, [rn, rm] -+- */ -+- off = offsetof(struct net_device, type); -+- emit_mov_i(ARM_R3, off, ctx); -+- emit(ARM_LDRH_R(r_A, r_scratch, ARM_R3), ctx); -+- } -++ case BPF_K: -++ /* Sign-extend immediate value to destination reg */ -++ emit_a32_mov_se_i64(is64, dst, imm, ctx); -+ break; -+- case BPF_ANC | SKF_AD_MARK: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); -+- off = offsetof(struct sk_buff, mark); -+- emit(ARM_LDR_I(r_A, r_skb, off), ctx); -+- break; -+- case BPF_ANC | SKF_AD_RXHASH: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); -+- off = offsetof(struct sk_buff, hash); -+- emit(ARM_LDR_I(r_A, r_skb, off), ctx); -+- break; -+- case BPF_ANC | SKF_AD_VLAN_TAG: -+- case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); -+- off = offsetof(struct sk_buff, vlan_tci); -+- emit(ARM_LDRH_I(r_A, r_skb, off), ctx); -+- if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) -+- OP_IMM3(ARM_AND, r_A, r_A, ~VLAN_TAG_PRESENT, ctx); -+- else { -+- OP_IMM3(ARM_LSR, r_A, r_A, 12, ctx); -+- OP_IMM3(ARM_AND, r_A, r_A, 0x1, ctx); -+- } -++ } -++ break; -++ /* dst = dst + src/imm */ -++ /* dst = dst - src/imm */ -++ /* dst = dst | src/imm */ -++ /* dst = dst & src/imm */ -++ /* dst = dst ^ src/imm */ -++ /* dst = dst * src/imm */ -++ /* dst = dst << src */ -++ /* dst = dst >> src */ -++ case BPF_ALU | BPF_ADD | BPF_K: -++ case BPF_ALU | BPF_ADD | BPF_X: -++ case BPF_ALU | BPF_SUB | BPF_K: -++ case BPF_ALU | BPF_SUB | BPF_X: -++ case BPF_ALU | BPF_OR | BPF_K: -++ case BPF_ALU | BPF_OR | BPF_X: -++ case BPF_ALU | BPF_AND | BPF_K: -++ case BPF_ALU | BPF_AND | BPF_X: -++ case BPF_ALU | BPF_XOR | BPF_K: -++ case BPF_ALU | BPF_XOR | BPF_X: -++ case BPF_ALU | BPF_MUL | BPF_K: -++ case BPF_ALU | BPF_MUL | BPF_X: -++ case BPF_ALU | BPF_LSH | BPF_X: -++ case BPF_ALU | BPF_RSH | BPF_X: -++ case BPF_ALU | BPF_ARSH | BPF_K: -++ case BPF_ALU | BPF_ARSH | BPF_X: -++ case BPF_ALU64 | BPF_ADD | BPF_K: -++ case BPF_ALU64 | BPF_ADD | BPF_X: -++ case BPF_ALU64 | BPF_SUB | BPF_K: -++ case BPF_ALU64 | BPF_SUB | BPF_X: -++ case BPF_ALU64 | BPF_OR | BPF_K: -++ case BPF_ALU64 | BPF_OR | BPF_X: -++ case BPF_ALU64 | BPF_AND | BPF_K: -++ case BPF_ALU64 | BPF_AND | BPF_X: -++ case BPF_ALU64 | BPF_XOR | BPF_K: -++ case BPF_ALU64 | BPF_XOR | BPF_X: -++ switch (BPF_SRC(code)) { -++ case BPF_X: -++ emit_a32_alu_r64(is64, dst, src, ctx, BPF_OP(code)); -++ break; -++ case BPF_K: -++ /* Move immediate value to the temporary register -++ * and then do the ALU operation on the temporary -++ * register as this will sign-extend the immediate -++ * value into temporary reg and then it would be -++ * safe to do the operation on it. -++ */ -++ emit_a32_mov_se_i64(is64, tmp2, imm, ctx); -++ emit_a32_alu_r64(is64, dst, tmp2, ctx, BPF_OP(code)); -+ break; -+- case BPF_ANC | SKF_AD_PKTTYPE: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, -+- __pkt_type_offset[0]) != 1); -+- off = PKT_TYPE_OFFSET(); -+- emit(ARM_LDRB_I(r_A, r_skb, off), ctx); -+- emit(ARM_AND_I(r_A, r_A, PKT_TYPE_MAX), ctx); -+-#ifdef __BIG_ENDIAN_BITFIELD -+- emit(ARM_LSR_I(r_A, r_A, 5), ctx); -+-#endif -++ } -++ break; -++ /* dst = dst / src(imm) */ -++ /* dst = dst % src(imm) */ -++ case BPF_ALU | BPF_DIV | BPF_K: -++ case BPF_ALU | BPF_DIV | BPF_X: -++ case BPF_ALU | BPF_MOD | BPF_K: -++ case BPF_ALU | BPF_MOD | BPF_X: -++ rd_lo = arm_bpf_get_reg32(dst_lo, tmp2[1], ctx); -++ switch (BPF_SRC(code)) { -++ case BPF_X: -++ rt = arm_bpf_get_reg32(src_lo, tmp2[0], ctx); -++ break; -++ case BPF_K: -++ rt = tmp2[0]; -++ emit_a32_mov_i(rt, imm, ctx); -+ break; -+- case BPF_ANC | SKF_AD_QUEUE: -+- ctx->seen |= SEEN_SKB; -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, -+- queue_mapping) != 2); -+- BUILD_BUG_ON(offsetof(struct sk_buff, -+- queue_mapping) > 0xff); -+- off = offsetof(struct sk_buff, queue_mapping); -+- emit(ARM_LDRH_I(r_A, r_skb, off), ctx); -+- break; -+- case BPF_ANC | SKF_AD_PAY_OFFSET: -+- ctx->seen |= SEEN_SKB | SEEN_CALL; -+- -+- emit(ARM_MOV_R(ARM_R0, r_skb), ctx); -+- emit_mov_i(ARM_R3, (unsigned int)skb_get_poff, ctx); -+- emit_blx_r(ARM_R3, ctx); -+- emit(ARM_MOV_R(r_A, ARM_R0), ctx); -+- break; -+- case BPF_LDX | BPF_W | BPF_ABS: -+- /* -+- * load a 32bit word from struct seccomp_data. -+- * seccomp_check_filter() will already have checked -+- * that k is 32bit aligned and lies within the -+- * struct seccomp_data. -++ default: -++ rt = src_lo; -++ break; -++ } -++ emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code)); -++ arm_bpf_put_reg32(dst_lo, rd_lo, ctx); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(dst_hi, 0, ctx); -++ break; -++ case BPF_ALU64 | BPF_DIV | BPF_K: -++ case BPF_ALU64 | BPF_DIV | BPF_X: -++ case BPF_ALU64 | BPF_MOD | BPF_K: -++ case BPF_ALU64 | BPF_MOD | BPF_X: -++ goto notyet; -++ /* dst = dst >> imm */ -++ /* dst = dst << imm */ -++ case BPF_ALU | BPF_RSH | BPF_K: -++ case BPF_ALU | BPF_LSH | BPF_K: -++ if (unlikely(imm > 31)) -++ return -EINVAL; -++ if (imm) -++ emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code)); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(dst_hi, 0, ctx); -++ break; -++ /* dst = dst << imm */ -++ case BPF_ALU64 | BPF_LSH | BPF_K: -++ if (unlikely(imm > 63)) -++ return -EINVAL; -++ emit_a32_lsh_i64(dst, imm, ctx); -++ break; -++ /* dst = dst >> imm */ -++ case BPF_ALU64 | BPF_RSH | BPF_K: -++ if (unlikely(imm > 63)) -++ return -EINVAL; -++ emit_a32_rsh_i64(dst, imm, ctx); -++ break; -++ /* dst = dst << src */ -++ case BPF_ALU64 | BPF_LSH | BPF_X: -++ emit_a32_lsh_r64(dst, src, ctx); -++ break; -++ /* dst = dst >> src */ -++ case BPF_ALU64 | BPF_RSH | BPF_X: -++ emit_a32_rsh_r64(dst, src, ctx); -++ break; -++ /* dst = dst >> src (signed) */ -++ case BPF_ALU64 | BPF_ARSH | BPF_X: -++ emit_a32_arsh_r64(dst, src, ctx); -++ break; -++ /* dst = dst >> imm (signed) */ -++ case BPF_ALU64 | BPF_ARSH | BPF_K: -++ if (unlikely(imm > 63)) -++ return -EINVAL; -++ emit_a32_arsh_i64(dst, imm, ctx); -++ break; -++ /* dst = ~dst */ -++ case BPF_ALU | BPF_NEG: -++ emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code)); -++ if (!ctx->prog->aux->verifier_zext) -++ emit_a32_mov_i(dst_hi, 0, ctx); -++ break; -++ /* dst = ~dst (64 bit) */ -++ case BPF_ALU64 | BPF_NEG: -++ emit_a32_neg64(dst, ctx); -++ break; -++ /* dst = dst * src/imm */ -++ case BPF_ALU64 | BPF_MUL | BPF_X: -++ case BPF_ALU64 | BPF_MUL | BPF_K: -++ switch (BPF_SRC(code)) { -++ case BPF_X: -++ emit_a32_mul_r64(dst, src, ctx); -++ break; -++ case BPF_K: -++ /* Move immediate value to the temporary register -++ * and then do the multiplication on it as this -++ * will sign-extend the immediate value into temp -++ * reg then it would be safe to do the operation -++ * on it. -+ */ -+- ctx->seen |= SEEN_SKB; -+- emit(ARM_LDR_I(r_A, r_skb, k), ctx); -++ emit_a32_mov_se_i64(is64, tmp2, imm, ctx); -++ emit_a32_mul_r64(dst, tmp2, ctx); -++ break; -++ } -++ break; -++ /* dst = htole(dst) */ -++ /* dst = htobe(dst) */ -++ case BPF_ALU | BPF_END | BPF_FROM_LE: -++ case BPF_ALU | BPF_END | BPF_FROM_BE: -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ if (BPF_SRC(code) == BPF_FROM_LE) -++ goto emit_bswap_uxt; -++ switch (imm) { -++ case 16: -++ emit_rev16(rd[1], rd[1], ctx); -++ goto emit_bswap_uxt; -++ case 32: -++ emit_rev32(rd[1], rd[1], ctx); -++ goto emit_bswap_uxt; -++ case 64: -++ emit_rev32(ARM_LR, rd[1], ctx); -++ emit_rev32(rd[1], rd[0], ctx); -++ emit(ARM_MOV_R(rd[0], ARM_LR), ctx); -+ break; -+- default: -+- return -1; -+ } -++ goto exit; -++emit_bswap_uxt: -++ switch (imm) { -++ case 16: -++ /* zero-extend 16 bits into 64 bits */ -++#if __LINUX_ARM_ARCH__ < 6 -++ emit_a32_mov_i(tmp2[1], 0xffff, ctx); -++ emit(ARM_AND_R(rd[1], rd[1], tmp2[1]), ctx); -++#else /* ARMv6+ */ -++ emit(ARM_UXTH(rd[1], rd[1]), ctx); -++#endif -++ if (!ctx->prog->aux->verifier_zext) -++ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); -++ break; -++ case 32: -++ /* zero-extend 32 bits into 64 bits */ -++ if (!ctx->prog->aux->verifier_zext) -++ emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx); -++ break; -++ case 64: -++ /* nop */ -++ break; -++ } -++exit: -++ arm_bpf_put_reg64(dst, rd, ctx); -++ break; -++ /* dst = imm64 */ -++ case BPF_LD | BPF_IMM | BPF_DW: -++ { -++ u64 val = (u32)imm | (u64)insn[1].imm << 32; -+ -+- if (ctx->flags & FLAG_IMM_OVERFLOW) -+- /* -+- * this instruction generated an overflow when -+- * trying to access the literal pool, so -+- * delegate this filter to the kernel interpreter. -+- */ -+- return -1; -++ emit_a32_mov_i64(dst, val, ctx); -++ -++ return 1; -+ } -++ /* LDX: dst = *(size *)(src + off) */ -++ case BPF_LDX | BPF_MEM | BPF_W: -++ case BPF_LDX | BPF_MEM | BPF_H: -++ case BPF_LDX | BPF_MEM | BPF_B: -++ case BPF_LDX | BPF_MEM | BPF_DW: -++ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ emit_ldx_r(dst, rn, off, ctx, BPF_SIZE(code)); -++ break; -++ /* ST: *(size *)(dst + off) = imm */ -++ case BPF_ST | BPF_MEM | BPF_W: -++ case BPF_ST | BPF_MEM | BPF_H: -++ case BPF_ST | BPF_MEM | BPF_B: -++ case BPF_ST | BPF_MEM | BPF_DW: -++ switch (BPF_SIZE(code)) { -++ case BPF_DW: -++ /* Sign-extend immediate value into temp reg */ -++ emit_a32_mov_se_i64(true, tmp2, imm, ctx); -++ break; -++ case BPF_W: -++ case BPF_H: -++ case BPF_B: -++ emit_a32_mov_i(tmp2[1], imm, ctx); -++ break; -++ } -++ emit_str_r(dst_lo, tmp2, off, ctx, BPF_SIZE(code)); -++ break; -++ /* STX XADD: lock *(u32 *)(dst + off) += src */ -++ case BPF_STX | BPF_XADD | BPF_W: -++ /* STX XADD: lock *(u64 *)(dst + off) += src */ -++ case BPF_STX | BPF_XADD | BPF_DW: -++ goto notyet; -++ /* STX: *(size *)(dst + off) = src */ -++ case BPF_STX | BPF_MEM | BPF_W: -++ case BPF_STX | BPF_MEM | BPF_H: -++ case BPF_STX | BPF_MEM | BPF_B: -++ case BPF_STX | BPF_MEM | BPF_DW: -++ rs = arm_bpf_get_reg64(src, tmp2, ctx); -++ emit_str_r(dst_lo, rs, off, ctx, BPF_SIZE(code)); -++ break; -++ /* PC += off if dst == src */ -++ /* PC += off if dst > src */ -++ /* PC += off if dst >= src */ -++ /* PC += off if dst < src */ -++ /* PC += off if dst <= src */ -++ /* PC += off if dst != src */ -++ /* PC += off if dst > src (signed) */ -++ /* PC += off if dst >= src (signed) */ -++ /* PC += off if dst < src (signed) */ -++ /* PC += off if dst <= src (signed) */ -++ /* PC += off if dst & src */ -++ case BPF_JMP | BPF_JEQ | BPF_X: -++ case BPF_JMP | BPF_JGT | BPF_X: -++ case BPF_JMP | BPF_JGE | BPF_X: -++ case BPF_JMP | BPF_JNE | BPF_X: -++ case BPF_JMP | BPF_JSGT | BPF_X: -++ case BPF_JMP | BPF_JSGE | BPF_X: -++ case BPF_JMP | BPF_JSET | BPF_X: -++ case BPF_JMP | BPF_JLE | BPF_X: -++ case BPF_JMP | BPF_JLT | BPF_X: -++ case BPF_JMP | BPF_JSLT | BPF_X: -++ case BPF_JMP | BPF_JSLE | BPF_X: -++ case BPF_JMP32 | BPF_JEQ | BPF_X: -++ case BPF_JMP32 | BPF_JGT | BPF_X: -++ case BPF_JMP32 | BPF_JGE | BPF_X: -++ case BPF_JMP32 | BPF_JNE | BPF_X: -++ case BPF_JMP32 | BPF_JSGT | BPF_X: -++ case BPF_JMP32 | BPF_JSGE | BPF_X: -++ case BPF_JMP32 | BPF_JSET | BPF_X: -++ case BPF_JMP32 | BPF_JLE | BPF_X: -++ case BPF_JMP32 | BPF_JLT | BPF_X: -++ case BPF_JMP32 | BPF_JSLT | BPF_X: -++ case BPF_JMP32 | BPF_JSLE | BPF_X: -++ /* Setup source registers */ -++ rm = arm_bpf_get_reg32(src_hi, tmp2[0], ctx); -++ rn = arm_bpf_get_reg32(src_lo, tmp2[1], ctx); -++ goto go_jmp; -++ /* PC += off if dst == imm */ -++ /* PC += off if dst > imm */ -++ /* PC += off if dst >= imm */ -++ /* PC += off if dst < imm */ -++ /* PC += off if dst <= imm */ -++ /* PC += off if dst != imm */ -++ /* PC += off if dst > imm (signed) */ -++ /* PC += off if dst >= imm (signed) */ -++ /* PC += off if dst < imm (signed) */ -++ /* PC += off if dst <= imm (signed) */ -++ /* PC += off if dst & imm */ -++ case BPF_JMP | BPF_JEQ | BPF_K: -++ case BPF_JMP | BPF_JGT | BPF_K: -++ case BPF_JMP | BPF_JGE | BPF_K: -++ case BPF_JMP | BPF_JNE | BPF_K: -++ case BPF_JMP | BPF_JSGT | BPF_K: -++ case BPF_JMP | BPF_JSGE | BPF_K: -++ case BPF_JMP | BPF_JSET | BPF_K: -++ case BPF_JMP | BPF_JLT | BPF_K: -++ case BPF_JMP | BPF_JLE | BPF_K: -++ case BPF_JMP | BPF_JSLT | BPF_K: -++ case BPF_JMP | BPF_JSLE | BPF_K: -++ case BPF_JMP32 | BPF_JEQ | BPF_K: -++ case BPF_JMP32 | BPF_JGT | BPF_K: -++ case BPF_JMP32 | BPF_JGE | BPF_K: -++ case BPF_JMP32 | BPF_JNE | BPF_K: -++ case BPF_JMP32 | BPF_JSGT | BPF_K: -++ case BPF_JMP32 | BPF_JSGE | BPF_K: -++ case BPF_JMP32 | BPF_JSET | BPF_K: -++ case BPF_JMP32 | BPF_JLT | BPF_K: -++ case BPF_JMP32 | BPF_JLE | BPF_K: -++ case BPF_JMP32 | BPF_JSLT | BPF_K: -++ case BPF_JMP32 | BPF_JSLE | BPF_K: -++ if (off == 0) -++ break; -++ rm = tmp2[0]; -++ rn = tmp2[1]; -++ /* Sign-extend immediate value */ -++ emit_a32_mov_se_i64(true, tmp2, imm, ctx); -++go_jmp: -++ /* Setup destination register */ -++ rd = arm_bpf_get_reg64(dst, tmp, ctx); -++ -++ /* Check for the condition */ -++ emit_ar_r(rd[0], rd[1], rm, rn, ctx, BPF_OP(code), -++ BPF_CLASS(code) == BPF_JMP); -++ -++ /* Setup JUMP instruction */ -++ jmp_offset = bpf2a32_offset(i+off, i, ctx); -++ switch (BPF_OP(code)) { -++ case BPF_JNE: -++ case BPF_JSET: -++ _emit(ARM_COND_NE, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JEQ: -++ _emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JGT: -++ _emit(ARM_COND_HI, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JGE: -++ _emit(ARM_COND_CS, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JSGT: -++ _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JSGE: -++ _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JLE: -++ _emit(ARM_COND_LS, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JLT: -++ _emit(ARM_COND_CC, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JSLT: -++ _emit(ARM_COND_LT, ARM_B(jmp_offset), ctx); -++ break; -++ case BPF_JSLE: -++ _emit(ARM_COND_GE, ARM_B(jmp_offset), ctx); -++ break; -++ } -++ break; -++ /* JMP OFF */ -++ case BPF_JMP | BPF_JA: -++ { -++ if (off == 0) -++ break; -++ jmp_offset = bpf2a32_offset(i+off, i, ctx); -++ check_imm24(jmp_offset); -++ emit(ARM_B(jmp_offset), ctx); -++ break; -++ } -++ /* tail call */ -++ case BPF_JMP | BPF_TAIL_CALL: -++ if (emit_bpf_tail_call(ctx)) -++ return -EFAULT; -++ break; -++ /* function call */ -++ case BPF_JMP | BPF_CALL: -++ { -++ const s8 *r0 = bpf2a32[BPF_REG_0]; -++ const s8 *r1 = bpf2a32[BPF_REG_1]; -++ const s8 *r2 = bpf2a32[BPF_REG_2]; -++ const s8 *r3 = bpf2a32[BPF_REG_3]; -++ const s8 *r4 = bpf2a32[BPF_REG_4]; -++ const s8 *r5 = bpf2a32[BPF_REG_5]; -++ const u32 func = (u32)__bpf_call_base + (u32)imm; -++ -++ emit_a32_mov_r64(true, r0, r1, ctx); -++ emit_a32_mov_r64(true, r1, r2, ctx); -++ emit_push_r64(r5, ctx); -++ emit_push_r64(r4, ctx); -++ emit_push_r64(r3, ctx); -+ -+- /* compute offsets only during the first pass */ -+- if (ctx->target == NULL) -+- ctx->offsets[i] = ctx->idx * 4; -++ emit_a32_mov_i(tmp[1], func, ctx); -++ emit_blx_r(tmp[1], ctx); -+ -++ emit(ARM_ADD_I(ARM_SP, ARM_SP, imm8m(24)), ctx); // callee clean -++ break; -++ } -++ /* function return */ -++ case BPF_JMP | BPF_EXIT: -++ /* Optimization: when last instruction is EXIT -++ * simply fallthrough to epilogue. -++ */ -++ if (i == ctx->prog->len - 1) -++ break; -++ jmp_offset = epilogue_offset(ctx); -++ check_imm24(jmp_offset); -++ emit(ARM_B(jmp_offset), ctx); -++ break; -++notyet: -++ pr_info_once("*** NOT YET: opcode %02x ***\n", code); -++ return -EFAULT; -++ default: -++ pr_err_once("unknown opcode %02x\n", code); -++ return -EINVAL; -++ } -++ -++ if (ctx->flags & FLAG_IMM_OVERFLOW) -++ /* -++ * this instruction generated an overflow when -++ * trying to access the literal pool, so -++ * delegate this filter to the kernel interpreter. -++ */ -++ return -1; -+ return 0; -+ } -+ -++static int build_body(struct jit_ctx *ctx) -++{ -++ const struct bpf_prog *prog = ctx->prog; -++ unsigned int i; -++ -++ for (i = 0; i < prog->len; i++) { -++ const struct bpf_insn *insn = &(prog->insnsi[i]); -++ int ret; -++ -++ ret = build_insn(insn, ctx); -++ -++ /* It's used with loading the 64 bit immediate value. */ -++ if (ret > 0) { -++ i++; -++ if (ctx->target == NULL) -++ ctx->offsets[i] = ctx->idx; -++ continue; -++ } -++ -++ if (ctx->target == NULL) -++ ctx->offsets[i] = ctx->idx; -++ -++ /* If unsuccesfull, return with error code */ -++ if (ret) -++ return ret; -++ } -++ return 0; -++} -++ -++static int validate_code(struct jit_ctx *ctx) -++{ -++ int i; -++ -++ for (i = 0; i < ctx->idx; i++) { -++ if (ctx->target[i] == __opcode_to_mem_arm(ARM_INST_UDF)) -++ return -1; -++ } -++ -++ return 0; -++} -+ -+-void bpf_jit_compile(struct bpf_prog *fp) -++void bpf_jit_compile(struct bpf_prog *prog) -+ { -++ /* Nothing to do here. We support Internal BPF. */ -++} -++ -++bool bpf_jit_needs_zext(void) -++{ -++ return true; -++} -++ -++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) -++{ -++ struct bpf_prog *tmp, *orig_prog = prog; -+ struct bpf_binary_header *header; -++ bool tmp_blinded = false; -+ struct jit_ctx ctx; -+- unsigned tmp_idx; -+- unsigned alloc_size; -+- u8 *target_ptr; -++ unsigned int tmp_idx; -++ unsigned int image_size; -++ u8 *image_ptr; -+ -+- if (!bpf_jit_enable) -+- return; -++ /* If BPF JIT was not enabled then we must fall back to -++ * the interpreter. -++ */ -++ if (!prog->jit_requested) -++ return orig_prog; -+ -+- memset(&ctx, 0, sizeof(ctx)); -+- ctx.skf = fp; -+- ctx.ret0_fp_idx = -1; -++ /* If constant blinding was enabled and we failed during blinding -++ * then we must fall back to the interpreter. Otherwise, we save -++ * the new JITed code. -++ */ -++ tmp = bpf_jit_blind_constants(prog); -+ -+- ctx.offsets = kzalloc(4 * (ctx.skf->len + 1), GFP_KERNEL); -+- if (ctx.offsets == NULL) -+- return; -++ if (IS_ERR(tmp)) -++ return orig_prog; -++ if (tmp != prog) { -++ tmp_blinded = true; -++ prog = tmp; -++ } -+ -+- /* fake pass to fill in the ctx->seen */ -+- if (unlikely(build_body(&ctx))) -++ memset(&ctx, 0, sizeof(ctx)); -++ ctx.prog = prog; -++ ctx.cpu_architecture = cpu_architecture(); -++ -++ /* Not able to allocate memory for offsets[] , then -++ * we must fall back to the interpreter -++ */ -++ ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL); -++ if (ctx.offsets == NULL) { -++ prog = orig_prog; -+ goto out; -++ } -++ -++ /* 1) fake pass to find in the length of the JITed code, -++ * to compute ctx->offsets and other context variables -++ * needed to compute final JITed code. -++ * Also, calculate random starting pointer/start of JITed code -++ * which is prefixed by random number of fault instructions. -++ * -++ * If the first pass fails then there is no chance of it -++ * being successful in the second pass, so just fall back -++ * to the interpreter. -++ */ -++ if (build_body(&ctx)) { -++ prog = orig_prog; -++ goto out_off; -++ } -+ -+ tmp_idx = ctx.idx; -+ build_prologue(&ctx); -+ ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; -+ -++ ctx.epilogue_offset = ctx.idx; -++ -+ #if __LINUX_ARM_ARCH__ < 7 -+ tmp_idx = ctx.idx; -+ build_epilogue(&ctx); -+@@ -1020,64 +1941,83 @@ void bpf_jit_compile(struct bpf_prog *fp -+ -+ ctx.idx += ctx.imm_count; -+ if (ctx.imm_count) { -+- ctx.imms = kzalloc(4 * ctx.imm_count, GFP_KERNEL); -+- if (ctx.imms == NULL) -+- goto out; -++ ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL); -++ if (ctx.imms == NULL) { -++ prog = orig_prog; -++ goto out_off; -++ } -+ } -+ #else -+- /* there's nothing after the epilogue on ARMv7 */ -++ /* there's nothing about the epilogue on ARMv7 */ -+ build_epilogue(&ctx); -+ #endif -+- alloc_size = 4 * ctx.idx; -+- header = bpf_jit_binary_alloc(alloc_size, &target_ptr, -+- 4, jit_fill_hole); -+- if (header == NULL) -+- goto out; -++ /* Now we can get the actual image size of the JITed arm code. -++ * Currently, we are not considering the THUMB-2 instructions -++ * for jit, although it can decrease the size of the image. -++ * -++ * As each arm instruction is of length 32bit, we are translating -++ * number of JITed intructions into the size required to store these -++ * JITed code. -++ */ -++ image_size = sizeof(u32) * ctx.idx; -+ -+- ctx.target = (u32 *) target_ptr; -++ /* Now we know the size of the structure to make */ -++ header = bpf_jit_binary_alloc(image_size, &image_ptr, -++ sizeof(u32), jit_fill_hole); -++ /* Not able to allocate memory for the structure then -++ * we must fall back to the interpretation -++ */ -++ if (header == NULL) { -++ prog = orig_prog; -++ goto out_imms; -++ } -++ -++ /* 2.) Actual pass to generate final JIT code */ -++ ctx.target = (u32 *) image_ptr; -+ ctx.idx = 0; -+ -+ build_prologue(&ctx); -++ -++ /* If building the body of the JITed code fails somehow, -++ * we fall back to the interpretation. -++ */ -+ if (build_body(&ctx) < 0) { -+-#if __LINUX_ARM_ARCH__ < 7 -+- if (ctx.imm_count) -+- kfree(ctx.imms); -+-#endif -++ image_ptr = NULL; -+ bpf_jit_binary_free(header); -+- goto out; -++ prog = orig_prog; -++ goto out_imms; -+ } -+ build_epilogue(&ctx); -+ -++ /* 3.) Extra pass to validate JITed Code */ -++ if (validate_code(&ctx)) { -++ image_ptr = NULL; -++ bpf_jit_binary_free(header); -++ prog = orig_prog; -++ goto out_imms; -++ } -+ flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx)); -+ -++ if (bpf_jit_enable > 1) -++ /* there are 2 passes here */ -++ bpf_jit_dump(prog->len, image_size, 2, ctx.target); -++ -++ bpf_jit_binary_lock_ro(header); -++ prog->bpf_func = (void *)ctx.target; -++ prog->jited = 1; -++ prog->jited_len = image_size; -++ -++out_imms: -+ #if __LINUX_ARM_ARCH__ < 7 -+ if (ctx.imm_count) -+ kfree(ctx.imms); -+ #endif -+- -+- if (bpf_jit_enable > 1) -+- /* there are 2 passes here */ -+- bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); -+- -+- set_memory_ro((unsigned long)header, header->pages); -+- fp->bpf_func = (void *)ctx.target; -+- fp->jited = 1; -+-out: -++out_off: -+ kfree(ctx.offsets); -+- return; -++out: -++ if (tmp_blinded) -++ bpf_jit_prog_release_other(prog, prog == orig_prog ? -++ tmp : orig_prog); -++ return prog; -+ } -+ -+-void bpf_jit_free(struct bpf_prog *fp) -+-{ -+- unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; -+- struct bpf_binary_header *header = (void *)addr; -+- -+- if (!fp->jited) -+- goto free_filter; -+- -+- set_memory_rw(addr, header->pages); -+- bpf_jit_binary_free(header); -+- -+-free_filter: -+- bpf_prog_unlock_free(fp); -+-} -+--- a/arch/arm/net/bpf_jit_32.h -++++ b/arch/arm/net/bpf_jit_32.h -+@@ -1,16 +1,14 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -+ /* -+ * Just-In-Time compiler for BPF filters on 32bit ARM -+ * -+ * Copyright (c) 2011 Mircea Gherzan -+- * -+- * This program is free software; you can redistribute it and/or modify it -+- * under the terms of the GNU General Public License as published by the -+- * Free Software Foundation; version 2 of the License. -+ */ -+ -+ #ifndef PFILTER_OPCODES_ARM_H -+ #define PFILTER_OPCODES_ARM_H -+ -++/* ARM 32bit Registers */ -+ #define ARM_R0 0 -+ #define ARM_R1 1 -+ #define ARM_R2 2 -+@@ -22,40 +20,46 @@ -+ #define ARM_R8 8 -+ #define ARM_R9 9 -+ #define ARM_R10 10 -+-#define ARM_FP 11 -+-#define ARM_IP 12 -+-#define ARM_SP 13 -+-#define ARM_LR 14 -+-#define ARM_PC 15 -+- -+-#define ARM_COND_EQ 0x0 -+-#define ARM_COND_NE 0x1 -+-#define ARM_COND_CS 0x2 -++#define ARM_FP 11 /* Frame Pointer */ -++#define ARM_IP 12 /* Intra-procedure scratch register */ -++#define ARM_SP 13 /* Stack pointer: as load/store base reg */ -++#define ARM_LR 14 /* Link Register */ -++#define ARM_PC 15 /* Program counter */ -++ -++#define ARM_COND_EQ 0x0 /* == */ -++#define ARM_COND_NE 0x1 /* != */ -++#define ARM_COND_CS 0x2 /* unsigned >= */ -+ #define ARM_COND_HS ARM_COND_CS -+-#define ARM_COND_CC 0x3 -++#define ARM_COND_CC 0x3 /* unsigned < */ -+ #define ARM_COND_LO ARM_COND_CC -+-#define ARM_COND_MI 0x4 -+-#define ARM_COND_PL 0x5 -+-#define ARM_COND_VS 0x6 -+-#define ARM_COND_VC 0x7 -+-#define ARM_COND_HI 0x8 -+-#define ARM_COND_LS 0x9 -+-#define ARM_COND_GE 0xa -+-#define ARM_COND_LT 0xb -+-#define ARM_COND_GT 0xc -+-#define ARM_COND_LE 0xd -+-#define ARM_COND_AL 0xe -++#define ARM_COND_MI 0x4 /* < 0 */ -++#define ARM_COND_PL 0x5 /* >= 0 */ -++#define ARM_COND_VS 0x6 /* Signed Overflow */ -++#define ARM_COND_VC 0x7 /* No Signed Overflow */ -++#define ARM_COND_HI 0x8 /* unsigned > */ -++#define ARM_COND_LS 0x9 /* unsigned <= */ -++#define ARM_COND_GE 0xa /* Signed >= */ -++#define ARM_COND_LT 0xb /* Signed < */ -++#define ARM_COND_GT 0xc /* Signed > */ -++#define ARM_COND_LE 0xd /* Signed <= */ -++#define ARM_COND_AL 0xe /* None */ -+ -+ /* register shift types */ -+ #define SRTYPE_LSL 0 -+ #define SRTYPE_LSR 1 -+ #define SRTYPE_ASR 2 -+ #define SRTYPE_ROR 3 -++#define SRTYPE_ASL (SRTYPE_LSL) -+ -+ #define ARM_INST_ADD_R 0x00800000 -++#define ARM_INST_ADDS_R 0x00900000 -++#define ARM_INST_ADC_R 0x00a00000 -++#define ARM_INST_ADC_I 0x02a00000 -+ #define ARM_INST_ADD_I 0x02800000 -++#define ARM_INST_ADDS_I 0x02900000 -+ -+ #define ARM_INST_AND_R 0x00000000 -++#define ARM_INST_ANDS_R 0x00100000 -+ #define ARM_INST_AND_I 0x02000000 -+ -+ #define ARM_INST_BIC_R 0x01c00000 -+@@ -71,13 +75,18 @@ -+ #define ARM_INST_EOR_R 0x00200000 -+ #define ARM_INST_EOR_I 0x02200000 -+ -+-#define ARM_INST_LDRB_I 0x05d00000 -++#define ARM_INST_LDST__U 0x00800000 -++#define ARM_INST_LDST__IMM12 0x00000fff -++#define ARM_INST_LDRB_I 0x05500000 -+ #define ARM_INST_LDRB_R 0x07d00000 -+-#define ARM_INST_LDRH_I 0x01d000b0 -++#define ARM_INST_LDRD_I 0x014000d0 -++#define ARM_INST_LDRH_I 0x015000b0 -+ #define ARM_INST_LDRH_R 0x019000b0 -+-#define ARM_INST_LDR_I 0x05900000 -++#define ARM_INST_LDR_I 0x05100000 -++#define ARM_INST_LDR_R 0x07900000 -+ -+ #define ARM_INST_LDM 0x08900000 -++#define ARM_INST_LDM_IA 0x08b00000 -+ -+ #define ARM_INST_LSL_I 0x01a00000 -+ #define ARM_INST_LSL_R 0x01a00010 -+@@ -86,6 +95,7 @@ -+ #define ARM_INST_LSR_R 0x01a00030 -+ -+ #define ARM_INST_MOV_R 0x01a00000 -++#define ARM_INST_MOVS_R 0x01b00000 -+ #define ARM_INST_MOV_I 0x03a00000 -+ #define ARM_INST_MOVW 0x03000000 -+ #define ARM_INST_MOVT 0x03400000 -+@@ -96,17 +106,29 @@ -+ #define ARM_INST_PUSH 0x092d0000 -+ -+ #define ARM_INST_ORR_R 0x01800000 -++#define ARM_INST_ORRS_R 0x01900000 -+ #define ARM_INST_ORR_I 0x03800000 -+ -+ #define ARM_INST_REV 0x06bf0f30 -+ #define ARM_INST_REV16 0x06bf0fb0 -+ -+ #define ARM_INST_RSB_I 0x02600000 -++#define ARM_INST_RSBS_I 0x02700000 -++#define ARM_INST_RSC_I 0x02e00000 -+ -+ #define ARM_INST_SUB_R 0x00400000 -++#define ARM_INST_SUBS_R 0x00500000 -++#define ARM_INST_RSB_R 0x00600000 -+ #define ARM_INST_SUB_I 0x02400000 -+- -+-#define ARM_INST_STR_I 0x05800000 -++#define ARM_INST_SUBS_I 0x02500000 -++#define ARM_INST_SBC_I 0x02c00000 -++#define ARM_INST_SBC_R 0x00c00000 -++#define ARM_INST_SBCS_R 0x00d00000 -++ -++#define ARM_INST_STR_I 0x05000000 -++#define ARM_INST_STRB_I 0x05400000 -++#define ARM_INST_STRD_I 0x014000f0 -++#define ARM_INST_STRH_I 0x014000b0 -+ -+ #define ARM_INST_TST_R 0x01100000 -+ #define ARM_INST_TST_I 0x03100000 -+@@ -117,6 +139,8 @@ -+ -+ #define ARM_INST_MLS 0x00600090 -+ -++#define ARM_INST_UXTH 0x06ff0070 -++ -+ /* -+ * Use a suitable undefined instruction to use for ARM/Thumb2 faulting. -+ * We need to be careful not to conflict with those used by other modules -+@@ -135,11 +159,18 @@ -+ #define _AL3_R(op, rd, rn, rm) ((op ## _R) | (rd) << 12 | (rn) << 16 | (rm)) -+ /* immediate */ -+ #define _AL3_I(op, rd, rn, imm) ((op ## _I) | (rd) << 12 | (rn) << 16 | (imm)) -++/* register with register-shift */ -++#define _AL3_SR(inst) (inst | (1 << 4)) -+ -+ #define ARM_ADD_R(rd, rn, rm) _AL3_R(ARM_INST_ADD, rd, rn, rm) -++#define ARM_ADDS_R(rd, rn, rm) _AL3_R(ARM_INST_ADDS, rd, rn, rm) -+ #define ARM_ADD_I(rd, rn, imm) _AL3_I(ARM_INST_ADD, rd, rn, imm) -++#define ARM_ADDS_I(rd, rn, imm) _AL3_I(ARM_INST_ADDS, rd, rn, imm) -++#define ARM_ADC_R(rd, rn, rm) _AL3_R(ARM_INST_ADC, rd, rn, rm) -++#define ARM_ADC_I(rd, rn, imm) _AL3_I(ARM_INST_ADC, rd, rn, imm) -+ -+ #define ARM_AND_R(rd, rn, rm) _AL3_R(ARM_INST_AND, rd, rn, rm) -++#define ARM_ANDS_R(rd, rn, rm) _AL3_R(ARM_INST_ANDS, rd, rn, rm) -+ #define ARM_AND_I(rd, rn, imm) _AL3_I(ARM_INST_AND, rd, rn, imm) -+ -+ #define ARM_BIC_R(rd, rn, rm) _AL3_R(ARM_INST_BIC, rd, rn, rm) -+@@ -155,27 +186,38 @@ -+ #define ARM_EOR_R(rd, rn, rm) _AL3_R(ARM_INST_EOR, rd, rn, rm) -+ #define ARM_EOR_I(rd, rn, imm) _AL3_I(ARM_INST_EOR, rd, rn, imm) -+ -+-#define ARM_LDR_I(rt, rn, off) (ARM_INST_LDR_I | (rt) << 12 | (rn) << 16 \ -+- | (off)) -+-#define ARM_LDRB_I(rt, rn, off) (ARM_INST_LDRB_I | (rt) << 12 | (rn) << 16 \ -+- | (off)) -+-#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | (rt) << 12 | (rn) << 16 \ -++#define ARM_LDR_R(rt, rn, rm) (ARM_INST_LDR_R | ARM_INST_LDST__U \ -++ | (rt) << 12 | (rn) << 16 \ -+ | (rm)) -+-#define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \ -+- | (((off) & 0xf0) << 4) | ((off) & 0xf)) -+-#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \ -++#define ARM_LDR_R_SI(rt, rn, rm, type, imm) \ -++ (ARM_INST_LDR_R | ARM_INST_LDST__U \ -++ | (rt) << 12 | (rn) << 16 \ -++ | (imm) << 7 | (type) << 5 | (rm)) -++#define ARM_LDRB_R(rt, rn, rm) (ARM_INST_LDRB_R | ARM_INST_LDST__U \ -++ | (rt) << 12 | (rn) << 16 \ -++ | (rm)) -++#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | ARM_INST_LDST__U \ -++ | (rt) << 12 | (rn) << 16 \ -+ | (rm)) -+ -+ #define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs)) -++#define ARM_LDM_IA(rn, regs) (ARM_INST_LDM_IA | (rn) << 16 | (regs)) -+ -+ #define ARM_LSL_R(rd, rn, rm) (_AL3_R(ARM_INST_LSL, rd, 0, rn) | (rm) << 8) -+ #define ARM_LSL_I(rd, rn, imm) (_AL3_I(ARM_INST_LSL, rd, 0, rn) | (imm) << 7) -+ -+ #define ARM_LSR_R(rd, rn, rm) (_AL3_R(ARM_INST_LSR, rd, 0, rn) | (rm) << 8) -+ #define ARM_LSR_I(rd, rn, imm) (_AL3_I(ARM_INST_LSR, rd, 0, rn) | (imm) << 7) -++#define ARM_ASR_R(rd, rn, rm) (_AL3_R(ARM_INST_ASR, rd, 0, rn) | (rm) << 8) -++#define ARM_ASR_I(rd, rn, imm) (_AL3_I(ARM_INST_ASR, rd, 0, rn) | (imm) << 7) -+ -+ #define ARM_MOV_R(rd, rm) _AL3_R(ARM_INST_MOV, rd, 0, rm) -++#define ARM_MOVS_R(rd, rm) _AL3_R(ARM_INST_MOVS, rd, 0, rm) -+ #define ARM_MOV_I(rd, imm) _AL3_I(ARM_INST_MOV, rd, 0, imm) -++#define ARM_MOV_SR(rd, rm, type, rs) \ -++ (_AL3_SR(ARM_MOV_R(rd, rm)) | (type) << 5 | (rs) << 8) -++#define ARM_MOV_SI(rd, rm, type, imm6) \ -++ (ARM_MOV_R(rd, rm) | (type) << 5 | (imm6) << 7) -+ -+ #define ARM_MOVW(rd, imm) \ -+ (ARM_INST_MOVW | ((imm) >> 12) << 16 | (rd) << 12 | ((imm) & 0x0fff)) -+@@ -190,19 +232,31 @@ -+ -+ #define ARM_ORR_R(rd, rn, rm) _AL3_R(ARM_INST_ORR, rd, rn, rm) -+ #define ARM_ORR_I(rd, rn, imm) _AL3_I(ARM_INST_ORR, rd, rn, imm) -+-#define ARM_ORR_S(rd, rn, rm, type, rs) \ -+- (ARM_ORR_R(rd, rn, rm) | (type) << 5 | (rs) << 7) -++#define ARM_ORR_SR(rd, rn, rm, type, rs) \ -++ (_AL3_SR(ARM_ORR_R(rd, rn, rm)) | (type) << 5 | (rs) << 8) -++#define ARM_ORRS_R(rd, rn, rm) _AL3_R(ARM_INST_ORRS, rd, rn, rm) -++#define ARM_ORRS_SR(rd, rn, rm, type, rs) \ -++ (_AL3_SR(ARM_ORRS_R(rd, rn, rm)) | (type) << 5 | (rs) << 8) -++#define ARM_ORR_SI(rd, rn, rm, type, imm6) \ -++ (ARM_ORR_R(rd, rn, rm) | (type) << 5 | (imm6) << 7) -++#define ARM_ORRS_SI(rd, rn, rm, type, imm6) \ -++ (ARM_ORRS_R(rd, rn, rm) | (type) << 5 | (imm6) << 7) -+ -+ #define ARM_REV(rd, rm) (ARM_INST_REV | (rd) << 12 | (rm)) -+ #define ARM_REV16(rd, rm) (ARM_INST_REV16 | (rd) << 12 | (rm)) -+ -+ #define ARM_RSB_I(rd, rn, imm) _AL3_I(ARM_INST_RSB, rd, rn, imm) -++#define ARM_RSBS_I(rd, rn, imm) _AL3_I(ARM_INST_RSBS, rd, rn, imm) -++#define ARM_RSC_I(rd, rn, imm) _AL3_I(ARM_INST_RSC, rd, rn, imm) -+ -+ #define ARM_SUB_R(rd, rn, rm) _AL3_R(ARM_INST_SUB, rd, rn, rm) -++#define ARM_SUBS_R(rd, rn, rm) _AL3_R(ARM_INST_SUBS, rd, rn, rm) -++#define ARM_RSB_R(rd, rn, rm) _AL3_R(ARM_INST_RSB, rd, rn, rm) -++#define ARM_SBC_R(rd, rn, rm) _AL3_R(ARM_INST_SBC, rd, rn, rm) -++#define ARM_SBCS_R(rd, rn, rm) _AL3_R(ARM_INST_SBCS, rd, rn, rm) -+ #define ARM_SUB_I(rd, rn, imm) _AL3_I(ARM_INST_SUB, rd, rn, imm) -+- -+-#define ARM_STR_I(rt, rn, off) (ARM_INST_STR_I | (rt) << 12 | (rn) << 16 \ -+- | (off)) -++#define ARM_SUBS_I(rd, rn, imm) _AL3_I(ARM_INST_SUBS, rd, rn, imm) -++#define ARM_SBC_I(rd, rn, imm) _AL3_I(ARM_INST_SBC, rd, rn, imm) -+ -+ #define ARM_TST_R(rn, rm) _AL3_R(ARM_INST_TST, 0, rn, rm) -+ #define ARM_TST_I(rn, imm) _AL3_I(ARM_INST_TST, 0, rn, imm) -+@@ -214,5 +268,6 @@ -+ -+ #define ARM_MLS(rd, rn, rm, ra) (ARM_INST_MLS | (rd) << 16 | (rn) | (rm) << 8 \ -+ | (ra) << 12) -++#define ARM_UXTH(rd, rm) (ARM_INST_UXTH | (rd) << 12 | (rm)) -+ -+ #endif /* PFILTER_OPCODES_ARM_H */ -+--- a/arch/arm/net/Makefile -++++ b/arch/arm/net/Makefile -+@@ -1,3 +1,4 @@ -++# SPDX-License-Identifier: GPL-2.0-only -+ # ARM-specific networking code -+ -+ obj-$(CONFIG_BPF_JIT) += bpf_jit_32.o -+--- /dev/null -++++ b/include/linux/bpf-cgroup.h -+@@ -0,0 +1,410 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++#ifndef _BPF_CGROUP_H -++#define _BPF_CGROUP_H -++ -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++struct sock; -++struct sockaddr; -++struct cgroup; -++struct sk_buff; -++struct bpf_map; -++struct bpf_prog; -++struct bpf_sock_ops_kern; -++struct bpf_cgroup_storage; -++struct ctl_table; -++struct ctl_table_header; -++ -++#ifdef CONFIG_CGROUP_BPF -++ -++extern struct static_key_false cgroup_bpf_enabled_key; -++#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key) -++ -++DECLARE_PER_CPU(struct bpf_cgroup_storage*, -++ bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); -++ -++#define for_each_cgroup_storage_type(stype) \ -++ for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) -++ -++struct bpf_cgroup_storage_map; -++ -++struct bpf_storage_buffer { -++ struct rcu_head rcu; -++ char data[0]; -++}; -++ -++struct bpf_cgroup_storage { -++ union { -++ struct bpf_storage_buffer *buf; -++ void __percpu *percpu_buf; -++ }; -++ struct bpf_cgroup_storage_map *map; -++ struct bpf_cgroup_storage_key key; -++ struct list_head list; -++ struct rb_node node; -++ struct rcu_head rcu; -++}; -++ -++struct bpf_prog_list { -++ struct list_head node; -++ struct bpf_prog *prog; -++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; -++}; -++ -++struct bpf_prog_array; -++ -++struct cgroup_bpf { -++ /* array of effective progs in this cgroup */ -++ struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE]; -++ -++ /* attached progs to this cgroup and attach flags -++ * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will -++ * have either zero or one element -++ * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS -++ */ -++ struct list_head progs[MAX_BPF_ATTACH_TYPE]; -++ u32 flags[MAX_BPF_ATTACH_TYPE]; -++ -++ /* temp storage for effective prog array used by prog_attach/detach */ -++ struct bpf_prog_array *inactive; -++ -++ /* reference counter used to detach bpf programs after cgroup removal */ -++ struct percpu_ref refcnt; -++ -++ /* cgroup_bpf is released using a work queue */ -++ struct work_struct release_work; -++}; -++ -++int cgroup_bpf_inherit(struct cgroup *cgrp); -++void cgroup_bpf_offline(struct cgroup *cgrp); -++ -++int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type, u32 flags); -++int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type); -++int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, -++ union bpf_attr __user *uattr); -++ -++/* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */ -++int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type, u32 flags); -++int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type, u32 flags); -++int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, -++ union bpf_attr __user *uattr); -++ -++int __cgroup_bpf_run_filter_skb(struct sock *sk, -++ struct sk_buff *skb, -++ enum bpf_attach_type type); -++ -++int __cgroup_bpf_run_filter_sk(struct sock *sk, -++ enum bpf_attach_type type); -++ -++int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, -++ struct sockaddr *uaddr, -++ enum bpf_attach_type type, -++ void *t_ctx); -++ -++int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, -++ struct bpf_sock_ops_kern *sock_ops, -++ enum bpf_attach_type type); -++ -++int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, -++ short access, enum bpf_attach_type type); -++ -++int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, -++ struct ctl_table *table, int write, -++ void __user *buf, size_t *pcount, -++ loff_t *ppos, void **new_buf, -++ enum bpf_attach_type type); -++ -++int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level, -++ int *optname, char __user *optval, -++ int *optlen, char **kernel_optval); -++int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, -++ int optname, char __user *optval, -++ int __user *optlen, int max_optlen, -++ int retval); -++ -++static inline enum bpf_cgroup_storage_type cgroup_storage_type( -++ struct bpf_map *map) -++{ -++ if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) -++ return BPF_CGROUP_STORAGE_PERCPU; -++ -++ return BPF_CGROUP_STORAGE_SHARED; -++} -++ -++static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage -++ *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) -++{ -++ enum bpf_cgroup_storage_type stype; -++ -++ for_each_cgroup_storage_type(stype) -++ this_cpu_write(bpf_cgroup_storage[stype], storage[stype]); -++} -++ -++struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, -++ enum bpf_cgroup_storage_type stype); -++void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage); -++void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, -++ struct cgroup *cgroup, -++ enum bpf_attach_type type); -++void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage); -++int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *map); -++void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *map); -++ -++int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value); -++int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, -++ void *value, u64 flags); -++ -++/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */ -++#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ -++ BPF_CGROUP_INET_INGRESS); \ -++ \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled && sk && sk == skb->sk) { \ -++ typeof(sk) __sk = sk_to_full_sk(sk); \ -++ if (sk_fullsock(__sk)) \ -++ __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \ -++ BPF_CGROUP_INET_EGRESS); \ -++ } \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_SK_PROG(sk, type) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) { \ -++ __ret = __cgroup_bpf_run_filter_sk(sk, type); \ -++ } \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) \ -++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE) -++ -++#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) \ -++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND) -++ -++#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) \ -++ BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND) -++ -++#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ -++ NULL); \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) { \ -++ lock_sock(sk); \ -++ __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ -++ t_ctx); \ -++ release_sock(sk); \ -++ } \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_BIND) -++ -++#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_BIND) -++ -++#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \ -++ sk->sk_prot->pre_connect) -++ -++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT) -++ -++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT) -++ -++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL) -++ -++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL) -++ -++#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx) -++ -++#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx) -++ -++#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL) -++ -++#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) \ -++ BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL) -++ -++#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled && (sock_ops)->sk) { \ -++ typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ -++ if (__sk && sk_fullsock(__sk)) \ -++ __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ -++ sock_ops, \ -++ BPF_CGROUP_SOCK_OPS); \ -++ } \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ -++ access, \ -++ BPF_CGROUP_DEVICE); \ -++ \ -++ __ret; \ -++}) -++ -++ -++#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos, nbuf) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ -++ buf, count, pos, nbuf, \ -++ BPF_CGROUP_SYSCTL); \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ -++ kernel_optval) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ -++ optname, optval, \ -++ optlen, \ -++ kernel_optval); \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \ -++({ \ -++ int __ret = 0; \ -++ if (cgroup_bpf_enabled) \ -++ get_user(__ret, optlen); \ -++ __ret; \ -++}) -++ -++#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen, \ -++ max_optlen, retval) \ -++({ \ -++ int __ret = retval; \ -++ if (cgroup_bpf_enabled) \ -++ __ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \ -++ optname, optval, \ -++ optlen, max_optlen, \ -++ retval); \ -++ __ret; \ -++}) -++ -++int cgroup_bpf_prog_attach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype, struct bpf_prog *prog); -++int cgroup_bpf_prog_detach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype); -++int cgroup_bpf_prog_query(const union bpf_attr *attr, -++ union bpf_attr __user *uattr); -++#else -++ -++struct bpf_prog; -++struct cgroup_bpf {}; -++static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; } -++static inline void cgroup_bpf_offline(struct cgroup *cgrp) {} -++ -++static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype, -++ struct bpf_prog *prog) -++{ -++ return -EINVAL; -++} -++ -++static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype) -++{ -++ return -EINVAL; -++} -++ -++static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ return -EINVAL; -++} -++ -++static inline void bpf_cgroup_storage_set( -++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {} -++static inline int bpf_cgroup_storage_assign(struct bpf_prog *prog, -++ struct bpf_map *map) { return 0; } -++static inline void bpf_cgroup_storage_release(struct bpf_prog *prog, -++ struct bpf_map *map) {} -++static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( -++ struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; } -++static inline void bpf_cgroup_storage_free( -++ struct bpf_cgroup_storage *storage) {} -++static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, -++ void *value) { -++ return 0; -++} -++static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, -++ void *key, void *value, u64 flags) { -++ return 0; -++} -++ -++#define cgroup_bpf_enabled (0) -++#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0) -++#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos,nbuf) ({ 0; }) -++#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; }) -++#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ -++ optlen, max_optlen, retval) ({ retval; }) -++#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ -++ kernel_optval) ({ 0; }) -++ -++#define for_each_cgroup_storage_type(stype) for (; false; ) -++ -++#endif /* CONFIG_CGROUP_BPF */ -++ -++#endif /* _BPF_CGROUP_H */ -+--- a/include/linux/bpf.h -++++ b/include/linux/bpf.h -+@@ -1,55 +1,183 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+ */ -+ #ifndef _LINUX_BPF_H -+ #define _LINUX_BPF_H 1 -+ -+ #include -++ -+ #include -+ #include -++#include -++#include -++#include -++#include -++#include -++#include -+ -++struct bpf_verifier_env; -++struct perf_event; -++struct bpf_prog; -+ struct bpf_map; -++struct sock; -++struct seq_file; -++struct btf; -++struct btf_type; -++ -++extern struct idr btf_idr; -++extern spinlock_t btf_idr_lock; -+ -+ /* map is generic key/value storage optionally accesible by eBPF programs */ -+ struct bpf_map_ops { -+ /* funcs callable from userspace (via syscall) */ -++ int (*map_alloc_check)(union bpf_attr *attr); -+ struct bpf_map *(*map_alloc)(union bpf_attr *attr); -+- void (*map_free)(struct bpf_map *); -++ void (*map_release)(struct bpf_map *map, struct file *map_file); -++ void (*map_free)(struct bpf_map *map); -+ int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); -++ void (*map_release_uref)(struct bpf_map *map); -++ void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); -+ -+ /* funcs callable from userspace and from eBPF programs */ -+ void *(*map_lookup_elem)(struct bpf_map *map, void *key); -+ int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); -+ int (*map_delete_elem)(struct bpf_map *map, void *key); -++ int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); -++ int (*map_pop_elem)(struct bpf_map *map, void *value); -++ int (*map_peek_elem)(struct bpf_map *map, void *value); -+ -+ /* funcs called by prog_array and perf_event_array map */ -+- void *(*map_fd_get_ptr) (struct bpf_map *map, int fd); -+- void (*map_fd_put_ptr) (void *ptr); -++ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, -++ int fd); -++ void (*map_fd_put_ptr)(void *ptr); -++ u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); -++ u32 (*map_fd_sys_lookup_elem)(void *ptr); -++ void (*map_seq_show_elem)(struct bpf_map *map, void *key, -++ struct seq_file *m); -++ int (*map_check_btf)(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type); -++ -++ /* Direct value access helpers. */ -++ int (*map_direct_value_addr)(const struct bpf_map *map, -++ u64 *imm, u32 off); -++ int (*map_direct_value_meta)(const struct bpf_map *map, -++ u64 imm, u32 *off); -++}; -++ -++struct bpf_map_memory { -++ u32 pages; -++ struct user_struct *user; -+ }; -+ -+ struct bpf_map { -+- atomic_t refcnt; -++ /* The first two cachelines with read-mostly members of which some -++ * are also accessed in fast-path (e.g. ops, max_entries). -++ */ -++ const struct bpf_map_ops *ops ____cacheline_aligned; -++ struct bpf_map *inner_map_meta; -++#ifdef CONFIG_SECURITY -++ void *security; -++#endif -+ enum bpf_map_type map_type; -+ u32 key_size; -+ u32 value_size; -+ u32 max_entries; -+- u32 pages; -++ u32 map_flags; -++ int spin_lock_off; /* >=0 valid offset, <0 error */ -++ u32 id; -++ int numa_node; -++ u32 btf_key_type_id; -++ u32 btf_value_type_id; -++ struct btf *btf; -++ struct bpf_map_memory memory; -+ bool unpriv_array; -+- struct user_struct *user; -+- const struct bpf_map_ops *ops; -+- struct work_struct work; -++ bool frozen; /* write-once */ -++ /* 48 bytes hole */ -++ -++ /* The 3rd and 4th cacheline with misc members to avoid false sharing -++ * particularly with refcounting. -++ */ -++ atomic_t refcnt ____cacheline_aligned; -+ atomic_t usercnt; -++ struct work_struct work; -++ char name[BPF_OBJ_NAME_LEN]; -+ }; -+ -+-struct bpf_map_type_list { -+- struct list_head list_node; -+- const struct bpf_map_ops *ops; -+- enum bpf_map_type type; -++static inline bool map_value_has_spin_lock(const struct bpf_map *map) -++{ -++ return map->spin_lock_off >= 0; -++} -++ -++static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) -++{ -++ if (likely(!map_value_has_spin_lock(map))) -++ return; -++ *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = -++ (struct bpf_spin_lock){}; -++} -++ -++/* copy everything but bpf_spin_lock */ -++static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) -++{ -++ if (unlikely(map_value_has_spin_lock(map))) { -++ u32 off = map->spin_lock_off; -++ -++ memcpy(dst, src, off); -++ memcpy(dst + off + sizeof(struct bpf_spin_lock), -++ src + off + sizeof(struct bpf_spin_lock), -++ map->value_size - off - sizeof(struct bpf_spin_lock)); -++ } else { -++ memcpy(dst, src, map->value_size); -++ } -++} -++void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, -++ bool lock_src); -++ -++struct bpf_offload_dev; -++struct bpf_offloaded_map; -++ -++struct bpf_map_dev_ops { -++ int (*map_get_next_key)(struct bpf_offloaded_map *map, -++ void *key, void *next_key); -++ int (*map_lookup_elem)(struct bpf_offloaded_map *map, -++ void *key, void *value); -++ int (*map_update_elem)(struct bpf_offloaded_map *map, -++ void *key, void *value, u64 flags); -++ int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); -+ }; -+ -++struct bpf_offloaded_map { -++ struct bpf_map map; -++ struct net_device *netdev; -++ const struct bpf_map_dev_ops *dev_ops; -++ void *dev_priv; -++ struct list_head offloads; -++}; -++ -++static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) -++{ -++ return container_of(map, struct bpf_offloaded_map, map); -++} -++ -++static inline bool bpf_map_offload_neutral(const struct bpf_map *map) -++{ -++ return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; -++} -++ -++static inline bool bpf_map_support_seq_show(const struct bpf_map *map) -++{ -++ return map->btf && map->ops->map_seq_show_elem; -++} -++ -++int map_check_no_btf(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type); -++ -++extern const struct bpf_map_ops bpf_map_offload_ops; -++ -+ /* function argument constraints */ -+ enum bpf_arg_type { -+ ARG_DONTCARE = 0, /* unused argument in helper function */ -+@@ -60,22 +188,40 @@ enum bpf_arg_type { -+ ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ -+ ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ -+ ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ -++ ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ -++ ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ -+ -+ /* the following constraints used to prototype bpf_memcmp() and other -+ * functions that access data on eBPF program stack -+ */ -+- ARG_PTR_TO_STACK, /* any pointer to eBPF program stack */ -+- ARG_CONST_STACK_SIZE, /* number of bytes accessed from stack */ -++ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ -++ ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ -++ ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, -++ * helper function must fill all bytes or clear -++ * them in error case. -++ */ -++ -++ ARG_CONST_SIZE, /* number of bytes accessed from memory */ -++ ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ -+ -+ ARG_PTR_TO_CTX, /* pointer to context */ -+ ARG_ANYTHING, /* any (initialized) argument is ok */ -++ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ -++ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ -++ ARG_PTR_TO_INT, /* pointer to int */ -++ ARG_PTR_TO_LONG, /* pointer to long */ -++ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ -+ }; -+ -+ /* type of values returned from helper functions */ -+ enum bpf_return_type { -+ RET_INTEGER, /* function returns integer */ -+ RET_VOID, /* function doesn't return anything */ -++ RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ -+ RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ -++ RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ -++ RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ -++ RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ -+ }; -+ -+ /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs -+@@ -85,6 +231,7 @@ enum bpf_return_type { -+ struct bpf_func_proto { -+ u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); -+ bool gpl_only; -++ bool pkt_access; -+ enum bpf_return_type ret_type; -+ enum bpf_arg_type arg1_type; -+ enum bpf_arg_type arg2_type; -+@@ -104,35 +251,172 @@ enum bpf_access_type { -+ BPF_WRITE = 2 -+ }; -+ -+-struct bpf_prog; -++/* types of values stored in eBPF registers */ -++/* Pointer types represent: -++ * pointer -++ * pointer + imm -++ * pointer + (u16) var -++ * pointer + (u16) var + imm -++ * if (range > 0) then [ptr, ptr + range - off) is safe to access -++ * if (id > 0) means that some 'var' was added -++ * if (off > 0) means that 'imm' was added -++ */ -++enum bpf_reg_type { -++ NOT_INIT = 0, /* nothing was written into register */ -++ SCALAR_VALUE, /* reg doesn't contain a valid pointer */ -++ PTR_TO_CTX, /* reg points to bpf_context */ -++ CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ -++ PTR_TO_MAP_VALUE, /* reg points to map element value */ -++ PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ -++ PTR_TO_STACK, /* reg == frame_pointer + offset */ -++ PTR_TO_PACKET_META, /* skb->data - meta_len */ -++ PTR_TO_PACKET, /* reg points to skb->data */ -++ PTR_TO_PACKET_END, /* skb->data + headlen */ -++ PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ -++ PTR_TO_SOCKET, /* reg points to struct bpf_sock */ -++ PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ -++ PTR_TO_SOCK_COMMON, /* reg points to sock_common */ -++ PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ -++ PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ -++ PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ -++ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ -++ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ -++}; -++ -++/* The information passed from prog-specific *_is_valid_access -++ * back to the verifier. -++ */ -++struct bpf_insn_access_aux { -++ enum bpf_reg_type reg_type; -++ int ctx_field_size; -++}; -++ -++static inline void -++bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) -++{ -++ aux->ctx_field_size = size; -++} -++ -++struct bpf_prog_ops { -++ int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, -++ union bpf_attr __user *uattr); -++}; -+ -+ struct bpf_verifier_ops { -+ /* return eBPF function prototype for verification */ -+- const struct bpf_func_proto *(*get_func_proto)(enum bpf_func_id func_id); -++ const struct bpf_func_proto * -++ (*get_func_proto)(enum bpf_func_id func_id, -++ const struct bpf_prog *prog); -+ -+ /* return true if 'size' wide access at offset 'off' within bpf_context -+ * with 'type' (read or write) is allowed -+ */ -+- bool (*is_valid_access)(int off, int size, enum bpf_access_type type); -++ bool (*is_valid_access)(int off, int size, enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info); -++ int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, -++ const struct bpf_prog *prog); -++ int (*gen_ld_abs)(const struct bpf_insn *orig, -++ struct bpf_insn *insn_buf); -++ u32 (*convert_ctx_access)(enum bpf_access_type type, -++ const struct bpf_insn *src, -++ struct bpf_insn *dst, -++ struct bpf_prog *prog, u32 *target_size); -++}; -+ -+- u32 (*convert_ctx_access)(enum bpf_access_type type, int dst_reg, -+- int src_reg, int ctx_off, -+- struct bpf_insn *insn, struct bpf_prog *prog); -++struct bpf_prog_offload_ops { -++ /* verifier basic callbacks */ -++ int (*insn_hook)(struct bpf_verifier_env *env, -++ int insn_idx, int prev_insn_idx); -++ int (*finalize)(struct bpf_verifier_env *env); -++ /* verifier optimization callbacks (called after .finalize) */ -++ int (*replace_insn)(struct bpf_verifier_env *env, u32 off, -++ struct bpf_insn *insn); -++ int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); -++ /* program management callbacks */ -++ int (*prepare)(struct bpf_prog *prog); -++ int (*translate)(struct bpf_prog *prog); -++ void (*destroy)(struct bpf_prog *prog); -+ }; -+ -+-struct bpf_prog_type_list { -+- struct list_head list_node; -+- const struct bpf_verifier_ops *ops; -+- enum bpf_prog_type type; -++struct bpf_prog_offload { -++ struct bpf_prog *prog; -++ struct net_device *netdev; -++ struct bpf_offload_dev *offdev; -++ void *dev_priv; -++ struct list_head offloads; -++ bool dev_state; -++ bool opt_failed; -++ void *jited_image; -++ u32 jited_len; -++}; -++ -++enum bpf_cgroup_storage_type { -++ BPF_CGROUP_STORAGE_SHARED, -++ BPF_CGROUP_STORAGE_PERCPU, -++ __BPF_CGROUP_STORAGE_MAX -++}; -++ -++#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX -++ -++struct bpf_prog_stats { -++ u64 cnt; -++ u64 nsecs; -++ struct u64_stats_sync syncp; -+ }; -+ -+ struct bpf_prog_aux { -+ atomic_t refcnt; -+ u32 used_map_cnt; -+- const struct bpf_verifier_ops *ops; -++ u32 max_ctx_offset; -++ u32 max_pkt_offset; -++ u32 max_tp_access; -++ u32 stack_depth; -++ u32 id; -++ u32 func_cnt; /* used by non-func prog as the number of func progs */ -++ u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ -++ bool verifier_zext; /* Zero extensions has been inserted by verifier. */ -++ bool offload_requested; -++ struct bpf_prog **func; -++ void *jit_data; /* JIT specific data. arch dependent */ -++ struct latch_tree_node ksym_tnode; -++ struct list_head ksym_lnode; -++ const struct bpf_prog_ops *ops; -+ struct bpf_map **used_maps; -+ struct bpf_prog *prog; -+ struct user_struct *user; -++ u64 load_time; /* ns since boottime */ -++ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; -++ char name[BPF_OBJ_NAME_LEN]; -++#ifdef CONFIG_SECURITY -++ void *security; -++#endif -++ struct bpf_prog_offload *offload; -++ struct btf *btf; -++ struct bpf_func_info *func_info; -++ /* bpf_line_info loaded from userspace. linfo->insn_off -++ * has the xlated insn offset. -++ * Both the main and sub prog share the same linfo. -++ * The subprog can access its first linfo by -++ * using the linfo_idx. -++ */ -++ struct bpf_line_info *linfo; -++ /* jited_linfo is the jited addr of the linfo. It has a -++ * one to one mapping to linfo: -++ * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. -++ * Both the main and sub prog share the same jited_linfo. -++ * The subprog can access its first jited_linfo by -++ * using the linfo_idx. -++ */ -++ void **jited_linfo; -++ u32 func_info_cnt; -++ u32 nr_linfo; -++ /* subprog can use linfo_idx to access its first linfo and -++ * jited_linfo. -++ * main prog always has linfo_idx == 0 -++ */ -++ u32 linfo_idx; -++ struct bpf_prog_stats __percpu *stats; -+ union { -+ struct work_struct work; -+ struct rcu_head rcu; -+@@ -153,76 +437,688 @@ struct bpf_array { -+ union { -+ char value[0] __aligned(8); -+ void *ptrs[0] __aligned(8); -++ void __percpu *pptrs[0] __aligned(8); -+ }; -+ }; -++ -++#define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ -+ #define MAX_TAIL_CALL_CNT 32 -+ -+-u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5); -+-void bpf_fd_array_map_clear(struct bpf_map *map); -++#define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ -++ BPF_F_RDONLY_PROG | \ -++ BPF_F_WRONLY | \ -++ BPF_F_WRONLY_PROG) -++ -++#define BPF_MAP_CAN_READ BIT(0) -++#define BPF_MAP_CAN_WRITE BIT(1) -++ -++static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) -++{ -++ u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); -++ -++ /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is -++ * not possible. -++ */ -++ if (access_flags & BPF_F_RDONLY_PROG) -++ return BPF_MAP_CAN_READ; -++ else if (access_flags & BPF_F_WRONLY_PROG) -++ return BPF_MAP_CAN_WRITE; -++ else -++ return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; -++} -++ -++static inline bool bpf_map_flags_access_ok(u32 access_flags) -++{ -++ return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != -++ (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); -++} -++ -++struct bpf_event_entry { -++ struct perf_event *event; -++ struct file *perf_file; -++ struct file *map_file; -++ struct rcu_head rcu; -++}; -++ -+ bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); -++int bpf_prog_calc_tag(struct bpf_prog *fp); -++ -+ const struct bpf_func_proto *bpf_get_trace_printk_proto(void); -+ -++typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, -++ unsigned long off, unsigned long len); -++typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, -++ const struct bpf_insn *src, -++ struct bpf_insn *dst, -++ struct bpf_prog *prog, -++ u32 *target_size); -++ -++u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, -++ void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); -++ -++/* an array of programs to be executed under rcu_lock. -++ * -++ * Typical usage: -++ * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); -++ * -++ * the structure returned by bpf_prog_array_alloc() should be populated -++ * with program pointers and the last pointer must be NULL. -++ * The user has to keep refcnt on the program and make sure the program -++ * is removed from the array before bpf_prog_put(). -++ * The 'struct bpf_prog_array *' should only be replaced with xchg() -++ * since other cpus are walking the array of pointers in parallel. -++ */ -++struct bpf_prog_array_item { -++ struct bpf_prog *prog; -++ struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; -++}; -++ -++struct bpf_prog_array { -++ struct rcu_head rcu; -++ struct bpf_prog_array_item items[0]; -++}; -++ -++struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); -++void bpf_prog_array_free(struct bpf_prog_array *progs); -++int bpf_prog_array_length(struct bpf_prog_array *progs); -++bool bpf_prog_array_is_empty(struct bpf_prog_array *array); -++int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, -++ __u32 __user *prog_ids, u32 cnt); -++ -++void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, -++ struct bpf_prog *old_prog); -++int bpf_prog_array_copy_info(struct bpf_prog_array *array, -++ u32 *prog_ids, u32 request_cnt, -++ u32 *prog_cnt); -++int bpf_prog_array_copy(struct bpf_prog_array *old_array, -++ struct bpf_prog *exclude_prog, -++ struct bpf_prog *include_prog, -++ struct bpf_prog_array **new_array); -++ -++#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null, set_cg_storage) \ -++ ({ \ -++ struct bpf_prog_array_item *_item; \ -++ struct bpf_prog *_prog; \ -++ struct bpf_prog_array *_array; \ -++ u32 _ret = 1; \ -++ preempt_disable(); \ -++ rcu_read_lock(); \ -++ _array = rcu_dereference(array); \ -++ if (unlikely(check_non_null && !_array))\ -++ goto _out; \ -++ _item = &_array->items[0]; \ -++ while ((_prog = READ_ONCE(_item->prog))) { \ -++ if (set_cg_storage) \ -++ bpf_cgroup_storage_set(_item->cgroup_storage); \ -++ _ret &= func(_prog, ctx); \ -++ _item++; \ -++ } \ -++_out: \ -++ rcu_read_unlock(); \ -++ preempt_enable(); \ -++ _ret; \ -++ }) -++ -++/* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs -++ * so BPF programs can request cwr for TCP packets. -++ * -++ * Current cgroup skb programs can only return 0 or 1 (0 to drop the -++ * packet. This macro changes the behavior so the low order bit -++ * indicates whether the packet should be dropped (0) or not (1) -++ * and the next bit is a congestion notification bit. This could be -++ * used by TCP to call tcp_enter_cwr() -++ * -++ * Hence, new allowed return values of CGROUP EGRESS BPF programs are: -++ * 0: drop packet -++ * 1: keep packet -++ * 2: drop packet and cn -++ * 3: keep packet and cn -++ * -++ * This macro then converts it to one of the NET_XMIT or an error -++ * code that is then interpreted as drop packet (and no cn): -++ * 0: NET_XMIT_SUCCESS skb should be transmitted -++ * 1: NET_XMIT_DROP skb should be dropped and cn -++ * 2: NET_XMIT_CN skb should be transmitted and cn -++ * 3: -EPERM skb should be dropped -++ */ -++#define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ -++ ({ \ -++ struct bpf_prog_array_item *_item; \ -++ struct bpf_prog *_prog; \ -++ struct bpf_prog_array *_array; \ -++ u32 ret; \ -++ u32 _ret = 1; \ -++ u32 _cn = 0; \ -++ preempt_disable(); \ -++ rcu_read_lock(); \ -++ _array = rcu_dereference(array); \ -++ _item = &_array->items[0]; \ -++ while ((_prog = READ_ONCE(_item->prog))) { \ -++ bpf_cgroup_storage_set(_item->cgroup_storage); \ -++ ret = func(_prog, ctx); \ -++ _ret &= (ret & 1); \ -++ _cn |= (ret & 2); \ -++ _item++; \ -++ } \ -++ rcu_read_unlock(); \ -++ preempt_enable(); \ -++ if (_ret) \ -++ _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ -++ else \ -++ _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ -++ _ret; \ -++ }) -++ -++#define BPF_PROG_RUN_ARRAY(array, ctx, func) \ -++ __BPF_PROG_RUN_ARRAY(array, ctx, func, false, true) -++ -++#define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ -++ __BPF_PROG_RUN_ARRAY(array, ctx, func, true, false) -++ -+ #ifdef CONFIG_BPF_SYSCALL -+-void bpf_register_prog_type(struct bpf_prog_type_list *tl); -+-void bpf_register_map_type(struct bpf_map_type_list *tl); -++DECLARE_PER_CPU(int, bpf_prog_active); -++ -++extern const struct file_operations bpf_map_fops; -++extern const struct file_operations bpf_prog_fops; -++ -++#define BPF_PROG_TYPE(_id, _name) \ -++ extern const struct bpf_prog_ops _name ## _prog_ops; \ -++ extern const struct bpf_verifier_ops _name ## _verifier_ops; -++#define BPF_MAP_TYPE(_id, _ops) \ -++ extern const struct bpf_map_ops _ops; -++#include -++#undef BPF_PROG_TYPE -++#undef BPF_MAP_TYPE -++ -++extern const struct bpf_prog_ops bpf_offload_prog_ops; -++extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; -++extern const struct bpf_verifier_ops xdp_analyzer_ops; -+ -+ struct bpf_prog *bpf_prog_get(u32 ufd); -+-struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog); -++struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, -++ bool attach_drv); -++struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i); -++void bpf_prog_sub(struct bpf_prog *prog, int i); -++struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog); -++struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); -+ void bpf_prog_put(struct bpf_prog *prog); -+-void bpf_prog_put_rcu(struct bpf_prog *prog); -++int __bpf_prog_charge(struct user_struct *user, u32 pages); -++void __bpf_prog_uncharge(struct user_struct *user, u32 pages); -++ -++void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); -++void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); -+ -+ struct bpf_map *bpf_map_get_with_uref(u32 ufd); -+ struct bpf_map *__bpf_map_get(struct fd f); -+-struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref); -++struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); -++struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map, -++ bool uref); -+ void bpf_map_put_with_uref(struct bpf_map *map); -+ void bpf_map_put(struct bpf_map *map); -++int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); -++void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); -++int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size); -++void bpf_map_charge_finish(struct bpf_map_memory *mem); -++void bpf_map_charge_move(struct bpf_map_memory *dst, -++ struct bpf_map_memory *src); -++void *bpf_map_area_alloc(u64 size, int numa_node); -++void bpf_map_area_free(void *base); -++void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); -+ -+ extern int sysctl_unprivileged_bpf_disabled; -+ -+-int bpf_map_new_fd(struct bpf_map *map); -++int bpf_map_new_fd(struct bpf_map *map, int flags); -+ int bpf_prog_new_fd(struct bpf_prog *prog); -+ -+ int bpf_obj_pin_user(u32 ufd, const char __user *pathname); -+-int bpf_obj_get_user(const char __user *pathname); -++int bpf_obj_get_user(const char __user *pathname, int flags); -++ -++int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); -++int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); -++int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, -++ u64 flags); -++int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, -++ u64 flags); -++ -++int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); -++ -++int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, -++ void *key, void *value, u64 map_flags); -++int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); -++int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, -++ void *key, void *value, u64 map_flags); -++int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); -++ -++int bpf_get_file_flag(int flags); -++int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, -++ size_t actual_size); -++ -++/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and -++ * forced to use 'long' read/writes to try to atomically copy long counters. -++ * Best-effort only. No barriers here, since it _will_ race with concurrent -++ * updates from BPF programs. Called from bpf syscall and mostly used with -++ * size 8 or 16 bytes, so ask compiler to inline it. -++ */ -++static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) -++{ -++ const long *lsrc = src; -++ long *ldst = dst; -++ -++ size /= sizeof(long); -++ while (size--) -++ *ldst++ = *lsrc++; -++} -+ -+ /* verify correctness of eBPF program */ -+-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr); -+-#else -+-static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl) -++int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, -++ union bpf_attr __user *uattr); -++ -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); -++#endif -++ -++/* Map specifics */ -++struct xdp_buff; -++struct sk_buff; -++ -++struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); -++struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); -++void __dev_map_flush(struct bpf_map *map); -++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, -++ struct net_device *dev_rx); -++int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, -++ struct bpf_prog *xdp_prog); -++ -++struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); -++void __cpu_map_flush(struct bpf_map *map); -++int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, -++ struct net_device *dev_rx); -++ -++/* Return map's numa specified by userspace */ -++static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) -+ { -++ return (attr->map_flags & BPF_F_NUMA_NODE) ? -++ attr->numa_node : NUMA_NO_NODE; -+ } -+ -++struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); -++int array_map_alloc_check(union bpf_attr *attr); -++ -++int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, -++ union bpf_attr __user *uattr); -++int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, -++ union bpf_attr __user *uattr); -++int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, -++ const union bpf_attr *kattr, -++ union bpf_attr __user *uattr); -++#else /* !CONFIG_BPF_SYSCALL */ -+ static inline struct bpf_prog *bpf_prog_get(u32 ufd) -+ { -+ return ERR_PTR(-EOPNOTSUPP); -+ } -+ -++static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, -++ enum bpf_prog_type type, -++ bool attach_drv) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, -++ int i) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++static inline void bpf_prog_sub(struct bpf_prog *prog, int i) -++{ -++} -++ -+ static inline void bpf_prog_put(struct bpf_prog *prog) -+ { -+ } -+ -+-static inline void bpf_prog_put_rcu(struct bpf_prog *prog) -++static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++static inline struct bpf_prog *__must_check -++bpf_prog_inc_not_zero(struct bpf_prog *prog) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) -++{ -++ return 0; -++} -++ -++static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) -++{ -++} -++ -++static inline int bpf_obj_get_user(const char __user *pathname, int flags) -++{ -++ return -EOPNOTSUPP; -++} -++ -++static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, -++ u32 key) -++{ -++ return NULL; -++} -++ -++static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, -++ u32 key) -++{ -++ return NULL; -++} -++ -++static inline void __dev_map_flush(struct bpf_map *map) -++{ -++} -++ -++struct xdp_buff; -++struct bpf_dtab_netdev; -++ -++static inline -++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, -++ struct net_device *dev_rx) -++{ -++ return 0; -++} -++ -++struct sk_buff; -++ -++static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, -++ struct sk_buff *skb, -++ struct bpf_prog *xdp_prog) -++{ -++ return 0; -++} -++ -++static inline -++struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) -++{ -++ return NULL; -++} -++ -++static inline void __cpu_map_flush(struct bpf_map *map) -++{ -++} -++ -++static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, -++ struct xdp_buff *xdp, -++ struct net_device *dev_rx) -++{ -++ return 0; -++} -++ -++static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, -++ enum bpf_prog_type type) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, -++ const union bpf_attr *kattr, -++ union bpf_attr __user *uattr) -++{ -++ return -ENOTSUPP; -++} -++ -++static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, -++ const union bpf_attr *kattr, -++ union bpf_attr __user *uattr) -++{ -++ return -ENOTSUPP; -++} -++ -++static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, -++ const union bpf_attr *kattr, -++ union bpf_attr __user *uattr) -++{ -++ return -ENOTSUPP; -++} -++#endif /* CONFIG_BPF_SYSCALL */ -++ -++static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, -++ enum bpf_prog_type type) -++{ -++ return bpf_prog_get_type_dev(ufd, type, false); -++} -++ -++bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); -++ -++#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) -++ -++static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) -++{ -++ return aux->offload_requested; -++} -++ -++static inline bool bpf_map_is_dev_bound(struct bpf_map *map) -++{ -++ return false; -++} -++ -++#else -++static inline int bpf_prog_offload_init(struct bpf_prog *prog, -++ union bpf_attr *attr) -++{ -++ return -EOPNOTSUPP; -++} -++ -++static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) -++{ -++ return false; -++} -++ -++static inline bool bpf_map_is_dev_bound(struct bpf_map *map) -++{ -++ return false; -++} -++ -++#endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ -++ -++#if defined(CONFIG_BPF_STREAM_PARSER) -++int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, -++ struct bpf_prog *old, u32 which); -++int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); -++int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); -++#else -++static inline int sock_map_prog_update(struct bpf_map *map, -++ struct bpf_prog *prog, -++ struct bpf_prog *old, u32 which) -+ { -++ return -EOPNOTSUPP; -++} -++ -++static inline int sock_map_get_from_fd(const union bpf_attr *attr, -++ struct bpf_prog *prog) -++{ -++ return -EINVAL; -++} -++ -++static inline int sock_map_prog_detach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype) -++{ -++ return -EOPNOTSUPP; -++} -++#endif -++ -++#if defined(CONFIG_XDP_SOCKETS) -++struct xdp_sock; -++struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key); -++int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, -++ struct xdp_sock *xs); -++void __xsk_map_flush(struct bpf_map *map); -++#else -++struct xdp_sock; -++static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, -++ u32 key) -++{ -++ return NULL; -++} -++ -++static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, -++ struct xdp_sock *xs) -++{ -++ return -EOPNOTSUPP; -++} -++ -++static inline void __xsk_map_flush(struct bpf_map *map) -++{ -++} -++#endif -++ -++#if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) -++void bpf_sk_reuseport_detach(struct sock *sk); -++int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, -++ void *value); -++int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 map_flags); -++#else -++static inline void bpf_sk_reuseport_detach(struct sock *sk) -++{ -++} -++ -++#ifdef CONFIG_BPF_SYSCALL -++static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, -++ void *key, void *value) -++{ -++ return -EOPNOTSUPP; -++} -++ -++static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, -++ void *key, void *value, -++ u64 map_flags) -++{ -++ return -EOPNOTSUPP; -+ } -+ #endif /* CONFIG_BPF_SYSCALL */ -++#endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ -+ -+ /* verifier prototypes for helper functions called from eBPF programs */ -+ extern const struct bpf_func_proto bpf_map_lookup_elem_proto; -+ extern const struct bpf_func_proto bpf_map_update_elem_proto; -+ extern const struct bpf_func_proto bpf_map_delete_elem_proto; -++extern const struct bpf_func_proto bpf_map_push_elem_proto; -++extern const struct bpf_func_proto bpf_map_pop_elem_proto; -++extern const struct bpf_func_proto bpf_map_peek_elem_proto; -+ -+ extern const struct bpf_func_proto bpf_get_prandom_u32_proto; -+ extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; -++extern const struct bpf_func_proto bpf_get_numa_node_id_proto; -+ extern const struct bpf_func_proto bpf_tail_call_proto; -+ extern const struct bpf_func_proto bpf_ktime_get_ns_proto; -+ extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; -+ extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; -+ extern const struct bpf_func_proto bpf_get_current_comm_proto; -+-extern const struct bpf_func_proto bpf_skb_vlan_push_proto; -+-extern const struct bpf_func_proto bpf_skb_vlan_pop_proto; -++extern const struct bpf_func_proto bpf_get_stackid_proto; -++extern const struct bpf_func_proto bpf_get_stack_proto; -++extern const struct bpf_func_proto bpf_sock_map_update_proto; -++extern const struct bpf_func_proto bpf_sock_hash_update_proto; -++extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; -++extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; -++extern const struct bpf_func_proto bpf_msg_redirect_map_proto; -++extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; -++extern const struct bpf_func_proto bpf_sk_redirect_map_proto; -++extern const struct bpf_func_proto bpf_spin_lock_proto; -++extern const struct bpf_func_proto bpf_spin_unlock_proto; -++extern const struct bpf_func_proto bpf_get_local_storage_proto; -++extern const struct bpf_func_proto bpf_strtol_proto; -++extern const struct bpf_func_proto bpf_strtoul_proto; -++extern const struct bpf_func_proto bpf_tcp_sock_proto; -+ -+ /* Shared helpers among cBPF and eBPF. */ -+ void bpf_user_rnd_init_once(void); -+ u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); -+ -++#if defined(CONFIG_NET) -++bool bpf_sock_common_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info); -++bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, -++ struct bpf_insn_access_aux *info); -++u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size); -++#else -++static inline bool bpf_sock_common_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++static inline bool bpf_sock_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++{ -++ return 0; -++} -++#endif -++ -++#ifdef CONFIG_INET -++bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, -++ struct bpf_insn_access_aux *info); -++ -++u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size); -++ -++bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, -++ struct bpf_insn_access_aux *info); -++ -++u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size); -++#else -++static inline bool bpf_tcp_sock_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++{ -++ return 0; -++} -++static inline bool bpf_xdp_sock_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++{ -++ return 0; -++} -++#endif /* CONFIG_INET */ -++ -+ #endif /* _LINUX_BPF_H */ -+--- /dev/null -++++ b/include/linux/bpf_trace.h -+@@ -0,0 +1,7 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++#ifndef __LINUX_BPF_TRACE_H__ -++#define __LINUX_BPF_TRACE_H__ -++ -++#include -++ -++#endif /* __LINUX_BPF_TRACE_H__ */ -+--- /dev/null -++++ b/include/linux/bpf_types.h -+@@ -0,0 +1,44 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++/* internal file - do not include directly */ -++ -++#ifdef CONFIG_NET -++BPF_PROG_TYPE(BPF_PROG_TYPE_SOCKET_FILTER, sk_filter) -++BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_CLS, tc_cls_act) -++BPF_PROG_TYPE(BPF_PROG_TYPE_SCHED_ACT, tc_cls_act) -++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_IN, lwt_in) -++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_OUT, lwt_out) -++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_XMIT, lwt_xmit) -++BPF_PROG_TYPE(BPF_PROG_TYPE_LWT_SEG6LOCAL, lwt_seg6local) -++BPF_PROG_TYPE(BPF_PROG_TYPE_SOCK_OPS, sock_ops) -++BPF_PROG_TYPE(BPF_PROG_TYPE_SK_SKB, sk_skb) -++BPF_PROG_TYPE(BPF_PROG_TYPE_SK_MSG, sk_msg) -++BPF_PROG_TYPE(BPF_PROG_TYPE_FLOW_DISSECTOR, flow_dissector) -++#endif -++#ifdef CONFIG_BPF_EVENTS -++BPF_PROG_TYPE(BPF_PROG_TYPE_KPROBE, kprobe) -++BPF_PROG_TYPE(BPF_PROG_TYPE_TRACEPOINT, tracepoint) -++BPF_PROG_TYPE(BPF_PROG_TYPE_PERF_EVENT, perf_event) -++BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT, raw_tracepoint) -++BPF_PROG_TYPE(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, raw_tracepoint_writable) -++#endif -++ -++BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_PROG_ARRAY, prog_array_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_PERF_EVENT_ARRAY, perf_event_array_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_HASH, htab_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_HASH, htab_percpu_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_HASH, htab_lru_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_LRU_PERCPU_HASH, htab_lru_percpu_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_LPM_TRIE, trie_map_ops) -++#ifdef CONFIG_PERF_EVENTS -++BPF_MAP_TYPE(BPF_MAP_TYPE_STACK_TRACE, stack_trace_map_ops) -++#endif -++BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_of_maps_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops) -++#ifdef CONFIG_NET -++BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP_HASH, dev_map_hash_ops) -++#endif -++BPF_MAP_TYPE(BPF_MAP_TYPE_QUEUE, queue_map_ops) -++BPF_MAP_TYPE(BPF_MAP_TYPE_STACK, stack_map_ops) -+--- /dev/null -++++ b/include/linux/bpf_verifier.h -+@@ -0,0 +1,425 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -++ */ -++#ifndef _LINUX_BPF_VERIFIER_H -++#define _LINUX_BPF_VERIFIER_H 1 -++ -++#include /* for enum bpf_reg_type */ -++#include /* for MAX_BPF_STACK */ -++#include -++ -++/* Maximum variable offset umax_value permitted when resolving memory accesses. -++ * In practice this is far bigger than any realistic pointer offset; this limit -++ * ensures that umax_value + (int)off + (int)size cannot overflow a u64. -++ */ -++#define BPF_MAX_VAR_OFF (1 << 29) -++/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures -++ * that converting umax_value to int cannot overflow. -++ */ -++#define BPF_MAX_VAR_SIZ (1 << 29) -++ -++/* Liveness marks, used for registers and spilled-regs (in stack slots). -++ * Read marks propagate upwards until they find a write mark; they record that -++ * "one of this state's descendants read this reg" (and therefore the reg is -++ * relevant for states_equal() checks). -++ * Write marks collect downwards and do not propagate; they record that "the -++ * straight-line code that reached this state (from its parent) wrote this reg" -++ * (and therefore that reads propagated from this state or its descendants -++ * should not propagate to its parent). -++ * A state with a write mark can receive read marks; it just won't propagate -++ * them to its parent, since the write mark is a property, not of the state, -++ * but of the link between it and its parent. See mark_reg_read() and -++ * mark_stack_slot_read() in kernel/bpf/verifier.c. -++ */ -++enum bpf_reg_liveness { -++ REG_LIVE_NONE = 0, /* reg hasn't been read or written this branch */ -++ REG_LIVE_READ32 = 0x1, /* reg was read, so we're sensitive to initial value */ -++ REG_LIVE_READ64 = 0x2, /* likewise, but full 64-bit content matters */ -++ REG_LIVE_READ = REG_LIVE_READ32 | REG_LIVE_READ64, -++ REG_LIVE_WRITTEN = 0x4, /* reg was written first, screening off later reads */ -++ REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */ -++}; -++ -++struct bpf_reg_state { -++ /* Ordering of fields matters. See states_equal() */ -++ enum bpf_reg_type type; -++ union { -++ /* valid when type == PTR_TO_PACKET */ -++ u16 range; -++ -++ /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | -++ * PTR_TO_MAP_VALUE_OR_NULL -++ */ -++ struct bpf_map *map_ptr; -++ -++ /* Max size from any of the above. */ -++ unsigned long raw; -++ }; -++ /* Fixed part of pointer offset, pointer types only */ -++ s32 off; -++ /* For PTR_TO_PACKET, used to find other pointers with the same variable -++ * offset, so they can share range knowledge. -++ * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we -++ * came from, when one is tested for != NULL. -++ * For PTR_TO_SOCKET this is used to share which pointers retain the -++ * same reference to the socket, to determine proper reference freeing. -++ */ -++ u32 id; -++ /* PTR_TO_SOCKET and PTR_TO_TCP_SOCK could be a ptr returned -++ * from a pointer-cast helper, bpf_sk_fullsock() and -++ * bpf_tcp_sock(). -++ * -++ * Consider the following where "sk" is a reference counted -++ * pointer returned from "sk = bpf_sk_lookup_tcp();": -++ * -++ * 1: sk = bpf_sk_lookup_tcp(); -++ * 2: if (!sk) { return 0; } -++ * 3: fullsock = bpf_sk_fullsock(sk); -++ * 4: if (!fullsock) { bpf_sk_release(sk); return 0; } -++ * 5: tp = bpf_tcp_sock(fullsock); -++ * 6: if (!tp) { bpf_sk_release(sk); return 0; } -++ * 7: bpf_sk_release(sk); -++ * 8: snd_cwnd = tp->snd_cwnd; // verifier will complain -++ * -++ * After bpf_sk_release(sk) at line 7, both "fullsock" ptr and -++ * "tp" ptr should be invalidated also. In order to do that, -++ * the reg holding "fullsock" and "sk" need to remember -++ * the original refcounted ptr id (i.e. sk_reg->id) in ref_obj_id -++ * such that the verifier can reset all regs which have -++ * ref_obj_id matching the sk_reg->id. -++ * -++ * sk_reg->ref_obj_id is set to sk_reg->id at line 1. -++ * sk_reg->id will stay as NULL-marking purpose only. -++ * After NULL-marking is done, sk_reg->id can be reset to 0. -++ * -++ * After "fullsock = bpf_sk_fullsock(sk);" at line 3, -++ * fullsock_reg->ref_obj_id is set to sk_reg->ref_obj_id. -++ * -++ * After "tp = bpf_tcp_sock(fullsock);" at line 5, -++ * tp_reg->ref_obj_id is set to fullsock_reg->ref_obj_id -++ * which is the same as sk_reg->ref_obj_id. -++ * -++ * From the verifier perspective, if sk, fullsock and tp -++ * are not NULL, they are the same ptr with different -++ * reg->type. In particular, bpf_sk_release(tp) is also -++ * allowed and has the same effect as bpf_sk_release(sk). -++ */ -++ u32 ref_obj_id; -++ /* For scalar types (SCALAR_VALUE), this represents our knowledge of -++ * the actual value. -++ * For pointer types, this represents the variable part of the offset -++ * from the pointed-to object, and is shared with all bpf_reg_states -++ * with the same id as us. -++ */ -++ struct tnum var_off; -++ /* Used to determine if any memory access using this register will -++ * result in a bad access. -++ * These refer to the same value as var_off, not necessarily the actual -++ * contents of the register. -++ */ -++ s64 smin_value; /* minimum possible (s64)value */ -++ s64 smax_value; /* maximum possible (s64)value */ -++ u64 umin_value; /* minimum possible (u64)value */ -++ u64 umax_value; /* maximum possible (u64)value */ -++ /* parentage chain for liveness checking */ -++ struct bpf_reg_state *parent; -++ /* Inside the callee two registers can be both PTR_TO_STACK like -++ * R1=fp-8 and R2=fp-8, but one of them points to this function stack -++ * while another to the caller's stack. To differentiate them 'frameno' -++ * is used which is an index in bpf_verifier_state->frame[] array -++ * pointing to bpf_func_state. -++ */ -++ u32 frameno; -++ /* Tracks subreg definition. The stored value is the insn_idx of the -++ * writing insn. This is safe because subreg_def is used before any insn -++ * patching which only happens after main verification finished. -++ */ -++ s32 subreg_def; -++ enum bpf_reg_liveness live; -++ /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ -++ bool precise; -++}; -++ -++enum bpf_stack_slot_type { -++ STACK_INVALID, /* nothing was stored in this stack slot */ -++ STACK_SPILL, /* register spilled into stack */ -++ STACK_MISC, /* BPF program wrote some data into this slot */ -++ STACK_ZERO, /* BPF program wrote constant zero */ -++}; -++ -++#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ -++ -++struct bpf_stack_state { -++ struct bpf_reg_state spilled_ptr; -++ u8 slot_type[BPF_REG_SIZE]; -++}; -++ -++struct bpf_reference_state { -++ /* Track each reference created with a unique id, even if the same -++ * instruction creates the reference multiple times (eg, via CALL). -++ */ -++ int id; -++ /* Instruction where the allocation of this reference occurred. This -++ * is used purely to inform the user of a reference leak. -++ */ -++ int insn_idx; -++}; -++ -++/* state of the program: -++ * type of all registers and stack info -++ */ -++struct bpf_func_state { -++ struct bpf_reg_state regs[MAX_BPF_REG]; -++ /* index of call instruction that called into this func */ -++ int callsite; -++ /* stack frame number of this function state from pov of -++ * enclosing bpf_verifier_state. -++ * 0 = main function, 1 = first callee. -++ */ -++ u32 frameno; -++ /* subprog number == index within subprog_stack_depth -++ * zero == main subprog -++ */ -++ u32 subprogno; -++ -++ /* The following fields should be last. See copy_func_state() */ -++ int acquired_refs; -++ struct bpf_reference_state *refs; -++ int allocated_stack; -++ struct bpf_stack_state *stack; -++}; -++ -++struct bpf_idx_pair { -++ u32 prev_idx; -++ u32 idx; -++}; -++ -++#define MAX_CALL_FRAMES 8 -++struct bpf_verifier_state { -++ /* call stack tracking */ -++ struct bpf_func_state *frame[MAX_CALL_FRAMES]; -++ struct bpf_verifier_state *parent; -++ /* -++ * 'branches' field is the number of branches left to explore: -++ * 0 - all possible paths from this state reached bpf_exit or -++ * were safely pruned -++ * 1 - at least one path is being explored. -++ * This state hasn't reached bpf_exit -++ * 2 - at least two paths are being explored. -++ * This state is an immediate parent of two children. -++ * One is fallthrough branch with branches==1 and another -++ * state is pushed into stack (to be explored later) also with -++ * branches==1. The parent of this state has branches==1. -++ * The verifier state tree connected via 'parent' pointer looks like: -++ * 1 -++ * 1 -++ * 2 -> 1 (first 'if' pushed into stack) -++ * 1 -++ * 2 -> 1 (second 'if' pushed into stack) -++ * 1 -++ * 1 -++ * 1 bpf_exit. -++ * -++ * Once do_check() reaches bpf_exit, it calls update_branch_counts() -++ * and the verifier state tree will look: -++ * 1 -++ * 1 -++ * 2 -> 1 (first 'if' pushed into stack) -++ * 1 -++ * 1 -> 1 (second 'if' pushed into stack) -++ * 0 -++ * 0 -++ * 0 bpf_exit. -++ * After pop_stack() the do_check() will resume at second 'if'. -++ * -++ * If is_state_visited() sees a state with branches > 0 it means -++ * there is a loop. If such state is exactly equal to the current state -++ * it's an infinite loop. Note states_equal() checks for states -++ * equvalency, so two states being 'states_equal' does not mean -++ * infinite loop. The exact comparison is provided by -++ * states_maybe_looping() function. It's a stronger pre-check and -++ * much faster than states_equal(). -++ * -++ * This algorithm may not find all possible infinite loops or -++ * loop iteration count may be too high. -++ * In such cases BPF_COMPLEXITY_LIMIT_INSNS limit kicks in. -++ */ -++ u32 branches; -++ u32 insn_idx; -++ u32 curframe; -++ u32 active_spin_lock; -++ bool speculative; -++ -++ /* first and last insn idx of this verifier state */ -++ u32 first_insn_idx; -++ u32 last_insn_idx; -++ /* jmp history recorded from first to last. -++ * backtracking is using it to go from last to first. -++ * For most states jmp_history_cnt is [0-3]. -++ * For loops can go up to ~40. -++ */ -++ struct bpf_idx_pair *jmp_history; -++ u32 jmp_history_cnt; -++}; -++ -++#define bpf_get_spilled_reg(slot, frame) \ -++ (((slot < frame->allocated_stack / BPF_REG_SIZE) && \ -++ (frame->stack[slot].slot_type[0] == STACK_SPILL)) \ -++ ? &frame->stack[slot].spilled_ptr : NULL) -++ -++/* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ -++#define bpf_for_each_spilled_reg(iter, frame, reg) \ -++ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame); \ -++ iter < frame->allocated_stack / BPF_REG_SIZE; \ -++ iter++, reg = bpf_get_spilled_reg(iter, frame)) -++ -++/* linked list of verifier states used to prune search */ -++struct bpf_verifier_state_list { -++ struct bpf_verifier_state state; -++ struct bpf_verifier_state_list *next; -++ int miss_cnt, hit_cnt; -++}; -++ -++/* Possible states for alu_state member. */ -++#define BPF_ALU_SANITIZE_SRC (1U << 0) -++#define BPF_ALU_SANITIZE_DST (1U << 1) -++#define BPF_ALU_NEG_VALUE (1U << 2) -++#define BPF_ALU_NON_POINTER (1U << 3) -++#define BPF_ALU_IMMEDIATE (1U << 4) -++#define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \ -++ BPF_ALU_SANITIZE_DST) -++ -++struct bpf_insn_aux_data { -++ union { -++ enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ -++ unsigned long map_state; /* pointer/poison value for maps */ -++ s32 call_imm; /* saved imm field of call insn */ -++ u32 alu_limit; /* limit for add/sub register with pointer */ -++ struct { -++ u32 map_index; /* index into used_maps[] */ -++ u32 map_off; /* offset from value base address */ -++ }; -++ }; -++ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ -++ int sanitize_stack_off; /* stack slot to be cleared */ -++ bool seen; /* this insn was processed by the verifier */ -++ bool zext_dst; /* this insn zero extends dst reg */ -++ u8 alu_state; /* used in combination with alu_limit */ -++ bool prune_point; -++ unsigned int orig_idx; /* original instruction index */ -++}; -++ -++#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ -++ -++#define BPF_VERIFIER_TMP_LOG_SIZE 1024 -++ -++struct bpf_verifier_log { -++ u32 level; -++ char kbuf[BPF_VERIFIER_TMP_LOG_SIZE]; -++ char __user *ubuf; -++ u32 len_used; -++ u32 len_total; -++}; -++ -++static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) -++{ -++ return log->len_used >= log->len_total - 1; -++} -++ -++#define BPF_LOG_LEVEL1 1 -++#define BPF_LOG_LEVEL2 2 -++#define BPF_LOG_STATS 4 -++#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2) -++#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS) -++ -++static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) -++{ -++ return log->level && log->ubuf && !bpf_verifier_log_full(log); -++} -++ -++#define BPF_MAX_SUBPROGS 256 -++ -++struct bpf_subprog_info { -++ u32 start; /* insn idx of function entry point */ -++ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ -++ u16 stack_depth; /* max. stack depth used by this function */ -++ bool has_tail_call; -++}; -++ -++/* single container for all structs -++ * one verifier_env per bpf_check() call -++ */ -++struct bpf_verifier_env { -++ u32 insn_idx; -++ u32 prev_insn_idx; -++ struct bpf_prog *prog; /* eBPF program being verified */ -++ const struct bpf_verifier_ops *ops; -++ struct bpf_verifier_stack_elem *head; /* stack of verifier states to be processed */ -++ int stack_size; /* number of states to be processed */ -++ bool strict_alignment; /* perform strict pointer alignment checks */ -++ bool test_state_freq; /* test verifier with different pruning frequency */ -++ struct bpf_verifier_state *cur_state; /* current verifier state */ -++ struct bpf_verifier_state_list **explored_states; /* search pruning optimization */ -++ struct bpf_verifier_state_list *free_list; -++ struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */ -++ u32 used_map_cnt; /* number of used maps */ -++ u32 id_gen; /* used to generate unique reg IDs */ -++ bool allow_ptr_leaks; -++ bool seen_direct_write; -++ struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ -++ const struct bpf_line_info *prev_linfo; -++ struct bpf_verifier_log log; -++ struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; -++ struct { -++ int *insn_state; -++ int *insn_stack; -++ int cur_stack; -++ } cfg; -++ u32 subprog_cnt; -++ /* number of instructions analyzed by the verifier */ -++ u32 prev_insn_processed, insn_processed; -++ /* number of jmps, calls, exits analyzed so far */ -++ u32 prev_jmps_processed, jmps_processed; -++ /* total verification time */ -++ u64 verification_time; -++ /* maximum number of verifier states kept in 'branching' instructions */ -++ u32 max_states_per_insn; -++ /* total number of allocated verifier states */ -++ u32 total_states; -++ /* some states are freed during program analysis. -++ * this is peak number of states. this number dominates kernel -++ * memory consumption during verification -++ */ -++ u32 peak_states; -++ /* longest register parentage chain walked for liveness marking */ -++ u32 longest_mark_read_walk; -++}; -++ -++__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, -++ const char *fmt, va_list args); -++__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, -++ const char *fmt, ...); -++ -++static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env) -++{ -++ struct bpf_verifier_state *cur = env->cur_state; -++ -++ return cur->frame[cur->curframe]; -++} -++ -++static inline struct bpf_reg_state *cur_regs(struct bpf_verifier_env *env) -++{ -++ return cur_func(env)->regs; -++} -++ -++int bpf_prog_offload_verifier_prep(struct bpf_prog *prog); -++int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env, -++ int insn_idx, int prev_insn_idx); -++int bpf_prog_offload_finalize(struct bpf_verifier_env *env); -++void -++bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off, -++ struct bpf_insn *insn); -++void -++bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); -++ -++#endif /* _LINUX_BPF_VERIFIER_H */ -+--- /dev/null -++++ b/include/linux/btf.h -+@@ -0,0 +1,72 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++/* Copyright (c) 2018 Facebook */ -++ -++#ifndef _LINUX_BTF_H -++#define _LINUX_BTF_H 1 -++ -++#include -++ -++struct btf; -++struct btf_member; -++struct btf_type; -++union bpf_attr; -++ -++extern const struct file_operations btf_fops; -++ -++void btf_put(struct btf *btf); -++int btf_new_fd(const union bpf_attr *attr); -++struct btf *btf_get_by_fd(int fd); -++int btf_get_info_by_fd(const struct btf *btf, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr); -++/* Figure out the size of a type_id. If type_id is a modifier -++ * (e.g. const), it will be resolved to find out the type with size. -++ * -++ * For example: -++ * In describing "const void *", type_id is "const" and "const" -++ * refers to "void *". The return type will be "void *". -++ * -++ * If type_id is a simple "int", then return type will be "int". -++ * -++ * @btf: struct btf object -++ * @type_id: Find out the size of type_id. The type_id of the return -++ * type is set to *type_id. -++ * @ret_size: It can be NULL. If not NULL, the size of the return -++ * type is set to *ret_size. -++ * Return: The btf_type (resolved to another type with size info if needed). -++ * NULL is returned if type_id itself does not have size info -++ * (e.g. void) or it cannot be resolved to another type that -++ * has size info. -++ * *type_id and *ret_size will not be changed in the -++ * NULL return case. -++ */ -++const struct btf_type *btf_type_id_size(const struct btf *btf, -++ u32 *type_id, -++ u32 *ret_size); -++void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, -++ struct seq_file *m); -++int btf_get_fd_by_id(u32 id); -++u32 btf_id(const struct btf *btf); -++bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, -++ const struct btf_member *m, -++ u32 expected_offset, u32 expected_size); -++int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t); -++bool btf_type_is_void(const struct btf_type *t); -++ -++#ifdef CONFIG_BPF_SYSCALL -++const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); -++const char *btf_name_by_offset(const struct btf *btf, u32 offset); -++#else -++static inline const struct btf_type *btf_type_by_id(const struct btf *btf, -++ u32 type_id) -++{ -++ return NULL; -++} -++static inline const char *btf_name_by_offset(const struct btf *btf, -++ u32 offset) -++{ -++ return NULL; -++} -++#endif -++ -++#endif -+--- a/include/uapi/linux/bpf_common.h -++++ b/include/uapi/linux/bpf_common.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+ #ifndef _UAPI__LINUX_BPF_COMMON_H__ -+ #define _UAPI__LINUX_BPF_COMMON_H__ -+ -+@@ -14,9 +15,10 @@ -+ -+ /* ld/ldx fields */ -+ #define BPF_SIZE(code) ((code) & 0x18) -+-#define BPF_W 0x00 -+-#define BPF_H 0x08 -+-#define BPF_B 0x10 -++#define BPF_W 0x00 /* 32-bit */ -++#define BPF_H 0x08 /* 16-bit */ -++#define BPF_B 0x10 /* 8-bit */ -++/* eBPF BPF_DW 0x18 64-bit */ -+ #define BPF_MODE(code) ((code) & 0xe0) -+ #define BPF_IMM 0x00 -+ #define BPF_ABS 0x20 -+--- a/include/uapi/linux/bpf.h -++++ b/include/uapi/linux/bpf.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+ * -+ * This program is free software; you can redistribute it and/or -+@@ -13,10 +14,11 @@ -+ /* Extended instruction set based on top of classic BPF */ -+ -+ /* instruction classes */ -++#define BPF_JMP32 0x06 /* jmp mode in word width */ -+ #define BPF_ALU64 0x07 /* alu mode in double word width */ -+ -+ /* ld/ldx fields */ -+-#define BPF_DW 0x18 /* double word */ -++#define BPF_DW 0x18 /* double word (64-bit) */ -+ #define BPF_XADD 0xc0 /* exclusive add */ -+ -+ /* alu/jmp fields */ -+@@ -30,9 +32,14 @@ -+ #define BPF_FROM_LE BPF_TO_LE -+ #define BPF_FROM_BE BPF_TO_BE -+ -++/* jmp encodings */ -+ #define BPF_JNE 0x50 /* jump != */ -++#define BPF_JLT 0xa0 /* LT is unsigned, '<' */ -++#define BPF_JLE 0xb0 /* LE is unsigned, '<=' */ -+ #define BPF_JSGT 0x60 /* SGT is signed '>', GT in x86 */ -+ #define BPF_JSGE 0x70 /* SGE is signed '>=', GE in x86 */ -++#define BPF_JSLT 0xc0 /* SLT is signed, '<' */ -++#define BPF_JSLE 0xd0 /* SLE is signed, '<=' */ -+ #define BPF_CALL 0x80 /* function call */ -+ #define BPF_EXIT 0x90 /* function return */ -+ -+@@ -63,6 +70,17 @@ struct bpf_insn { -+ __s32 imm; /* signed immediate constant */ -+ }; -+ -++/* Key of an a BPF_MAP_TYPE_LPM_TRIE entry */ -++struct bpf_lpm_trie_key { -++ __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6 */ -++ __u8 data[0]; /* Arbitrary size */ -++}; -++ -++struct bpf_cgroup_storage_key { -++ __u64 cgroup_inode_id; /* cgroup inode id */ -++ __u32 attach_type; /* program attach type */ -++}; -++ -+ /* BPF syscall commands, see bpf(2) man-page for details. */ -+ enum bpf_cmd { -+ BPF_MAP_CREATE, -+@@ -73,6 +91,22 @@ enum bpf_cmd { -+ BPF_PROG_LOAD, -+ BPF_OBJ_PIN, -+ BPF_OBJ_GET, -++ BPF_PROG_ATTACH, -++ BPF_PROG_DETACH, -++ BPF_PROG_TEST_RUN, -++ BPF_PROG_GET_NEXT_ID, -++ BPF_MAP_GET_NEXT_ID, -++ BPF_PROG_GET_FD_BY_ID, -++ BPF_MAP_GET_FD_BY_ID, -++ BPF_OBJ_GET_INFO_BY_FD, -++ BPF_PROG_QUERY, -++ BPF_RAW_TRACEPOINT_OPEN, -++ BPF_BTF_LOAD, -++ BPF_BTF_GET_FD_BY_ID, -++ BPF_TASK_FD_QUERY, -++ BPF_MAP_LOOKUP_AND_DELETE_ELEM, -++ BPF_MAP_FREEZE, -++ BPF_BTF_GET_NEXT_ID, -+ }; -+ -+ enum bpf_map_type { -+@@ -81,22 +115,256 @@ enum bpf_map_type { -+ BPF_MAP_TYPE_ARRAY, -+ BPF_MAP_TYPE_PROG_ARRAY, -+ BPF_MAP_TYPE_PERF_EVENT_ARRAY, -++ BPF_MAP_TYPE_PERCPU_HASH, -++ BPF_MAP_TYPE_PERCPU_ARRAY, -++ BPF_MAP_TYPE_STACK_TRACE, -++ BPF_MAP_TYPE_CGROUP_ARRAY, -++ BPF_MAP_TYPE_LRU_HASH, -++ BPF_MAP_TYPE_LRU_PERCPU_HASH, -++ BPF_MAP_TYPE_LPM_TRIE, -++ BPF_MAP_TYPE_ARRAY_OF_MAPS, -++ BPF_MAP_TYPE_HASH_OF_MAPS, -++ BPF_MAP_TYPE_DEVMAP, -++ BPF_MAP_TYPE_SOCKMAP, -++ BPF_MAP_TYPE_CPUMAP, -++ BPF_MAP_TYPE_XSKMAP, -++ BPF_MAP_TYPE_SOCKHASH, -++ BPF_MAP_TYPE_CGROUP_STORAGE, -++ BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, -++ BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, -++ BPF_MAP_TYPE_QUEUE, -++ BPF_MAP_TYPE_STACK, -++ BPF_MAP_TYPE_SK_STORAGE, -++ BPF_MAP_TYPE_DEVMAP_HASH, -+ }; -+ -++/* Note that tracing related programs such as -++ * BPF_PROG_TYPE_{KPROBE,TRACEPOINT,PERF_EVENT,RAW_TRACEPOINT} -++ * are not subject to a stable API since kernel internal data -++ * structures can change from release to release and may -++ * therefore break existing tracing BPF programs. Tracing BPF -++ * programs correspond to /a/ specific kernel which is to be -++ * analyzed, and not /a/ specific kernel /and/ all future ones. -++ */ -+ enum bpf_prog_type { -+ BPF_PROG_TYPE_UNSPEC, -+ BPF_PROG_TYPE_SOCKET_FILTER, -+ BPF_PROG_TYPE_KPROBE, -+ BPF_PROG_TYPE_SCHED_CLS, -+ BPF_PROG_TYPE_SCHED_ACT, -++ BPF_PROG_TYPE_TRACEPOINT, -++ BPF_PROG_TYPE_XDP, -++ BPF_PROG_TYPE_PERF_EVENT, -++ BPF_PROG_TYPE_CGROUP_SKB, -++ BPF_PROG_TYPE_CGROUP_SOCK, -++ BPF_PROG_TYPE_LWT_IN, -++ BPF_PROG_TYPE_LWT_OUT, -++ BPF_PROG_TYPE_LWT_XMIT, -++ BPF_PROG_TYPE_SOCK_OPS, -++ BPF_PROG_TYPE_SK_SKB, -++ BPF_PROG_TYPE_CGROUP_DEVICE, -++ BPF_PROG_TYPE_SK_MSG, -++ BPF_PROG_TYPE_RAW_TRACEPOINT, -++ BPF_PROG_TYPE_CGROUP_SOCK_ADDR, -++ BPF_PROG_TYPE_LWT_SEG6LOCAL, -++ BPF_PROG_TYPE_LIRC_MODE2, -++ BPF_PROG_TYPE_SK_REUSEPORT, -++ BPF_PROG_TYPE_FLOW_DISSECTOR, -++ BPF_PROG_TYPE_CGROUP_SYSCTL, -++ BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, -++ BPF_PROG_TYPE_CGROUP_SOCKOPT, -+ }; -+ -++enum bpf_attach_type { -++ BPF_CGROUP_INET_INGRESS, -++ BPF_CGROUP_INET_EGRESS, -++ BPF_CGROUP_INET_SOCK_CREATE, -++ BPF_CGROUP_SOCK_OPS, -++ BPF_SK_SKB_STREAM_PARSER, -++ BPF_SK_SKB_STREAM_VERDICT, -++ BPF_CGROUP_DEVICE, -++ BPF_SK_MSG_VERDICT, -++ BPF_CGROUP_INET4_BIND, -++ BPF_CGROUP_INET6_BIND, -++ BPF_CGROUP_INET4_CONNECT, -++ BPF_CGROUP_INET6_CONNECT, -++ BPF_CGROUP_INET4_POST_BIND, -++ BPF_CGROUP_INET6_POST_BIND, -++ BPF_CGROUP_UDP4_SENDMSG, -++ BPF_CGROUP_UDP6_SENDMSG, -++ BPF_LIRC_MODE2, -++ BPF_FLOW_DISSECTOR, -++ BPF_CGROUP_SYSCTL, -++ BPF_CGROUP_UDP4_RECVMSG, -++ BPF_CGROUP_UDP6_RECVMSG, -++ BPF_CGROUP_GETSOCKOPT, -++ BPF_CGROUP_SETSOCKOPT, -++ __MAX_BPF_ATTACH_TYPE -++}; -++ -++#define MAX_BPF_ATTACH_TYPE __MAX_BPF_ATTACH_TYPE -++ -++/* cgroup-bpf attach flags used in BPF_PROG_ATTACH command -++ * -++ * NONE(default): No further bpf programs allowed in the subtree. -++ * -++ * BPF_F_ALLOW_OVERRIDE: If a sub-cgroup installs some bpf program, -++ * the program in this cgroup yields to sub-cgroup program. -++ * -++ * BPF_F_ALLOW_MULTI: If a sub-cgroup installs some bpf program, -++ * that cgroup program gets run in addition to the program in this cgroup. -++ * -++ * Only one program is allowed to be attached to a cgroup with -++ * NONE or BPF_F_ALLOW_OVERRIDE flag. -++ * Attaching another program on top of NONE or BPF_F_ALLOW_OVERRIDE will -++ * release old program and attach the new one. Attach flags has to match. -++ * -++ * Multiple programs are allowed to be attached to a cgroup with -++ * BPF_F_ALLOW_MULTI flag. They are executed in FIFO order -++ * (those that were attached first, run first) -++ * The programs of sub-cgroup are executed first, then programs of -++ * this cgroup and then programs of parent cgroup. -++ * When children program makes decision (like picking TCP CA or sock bind) -++ * parent program has a chance to override it. -++ * -++ * A cgroup with MULTI or OVERRIDE flag allows any attach flags in sub-cgroups. -++ * A cgroup with NONE doesn't allow any programs in sub-cgroups. -++ * Ex1: -++ * cgrp1 (MULTI progs A, B) -> -++ * cgrp2 (OVERRIDE prog C) -> -++ * cgrp3 (MULTI prog D) -> -++ * cgrp4 (OVERRIDE prog E) -> -++ * cgrp5 (NONE prog F) -++ * the event in cgrp5 triggers execution of F,D,A,B in that order. -++ * if prog F is detached, the execution is E,D,A,B -++ * if prog F and D are detached, the execution is E,A,B -++ * if prog F, E and D are detached, the execution is C,A,B -++ * -++ * All eligible programs are executed regardless of return code from -++ * earlier programs. -++ */ -++#define BPF_F_ALLOW_OVERRIDE (1U << 0) -++#define BPF_F_ALLOW_MULTI (1U << 1) -++ -++/* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the -++ * verifier will perform strict alignment checking as if the kernel -++ * has been built with CONFIG_EFFICIENT_UNALIGNED_ACCESS not set, -++ * and NET_IP_ALIGN defined to 2. -++ */ -++#define BPF_F_STRICT_ALIGNMENT (1U << 0) -++ -++/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the -++ * verifier will allow any alignment whatsoever. On platforms -++ * with strict alignment requirements for loads ands stores (such -++ * as sparc and mips) the verifier validates that all loads and -++ * stores provably follow this requirement. This flag turns that -++ * checking and enforcement off. -++ * -++ * It is mostly used for testing when we want to validate the -++ * context and memory access aspects of the verifier, but because -++ * of an unaligned access the alignment check would trigger before -++ * the one we are interested in. -++ */ -++#define BPF_F_ANY_ALIGNMENT (1U << 1) -++ -++/* BPF_F_TEST_RND_HI32 is used in BPF_PROG_LOAD command for testing purpose. -++ * Verifier does sub-register def/use analysis and identifies instructions whose -++ * def only matters for low 32-bit, high 32-bit is never referenced later -++ * through implicit zero extension. Therefore verifier notifies JIT back-ends -++ * that it is safe to ignore clearing high 32-bit for these instructions. This -++ * saves some back-ends a lot of code-gen. However such optimization is not -++ * necessary on some arches, for example x86_64, arm64 etc, whose JIT back-ends -++ * hence hasn't used verifier's analysis result. But, we really want to have a -++ * way to be able to verify the correctness of the described optimization on -++ * x86_64 on which testsuites are frequently exercised. -++ * -++ * So, this flag is introduced. Once it is set, verifier will randomize high -++ * 32-bit for those instructions who has been identified as safe to ignore them. -++ * Then, if verifier is not doing correct analysis, such randomization will -++ * regress tests to expose bugs. -++ */ -++#define BPF_F_TEST_RND_HI32 (1U << 2) -++ -++/* The verifier internal test flag. Behavior is undefined */ -++#define BPF_F_TEST_STATE_FREQ (1U << 3) -++ -++/* When BPF ldimm64's insn[0].src_reg != 0 then this can have -++ * two extensions: -++ * -++ * insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE -++ * insn[0].imm: map fd map fd -++ * insn[1].imm: 0 offset into value -++ * insn[0].off: 0 0 -++ * insn[1].off: 0 0 -++ * ldimm64 rewrite: address of map address of map[0]+offset -++ * verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE -++ */ -+ #define BPF_PSEUDO_MAP_FD 1 -++#define BPF_PSEUDO_MAP_VALUE 2 -++ -++/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative -++ * offset to another bpf function -++ */ -++#define BPF_PSEUDO_CALL 1 -+ -+ /* flags for BPF_MAP_UPDATE_ELEM command */ -+ #define BPF_ANY 0 /* create new element or update existing */ -+ #define BPF_NOEXIST 1 /* create new element if it didn't exist */ -+ #define BPF_EXIST 2 /* update existing element */ -++#define BPF_F_LOCK 4 /* spin_lock-ed map_lookup/map_update */ -++ -++/* flags for BPF_MAP_CREATE command */ -++#define BPF_F_NO_PREALLOC (1U << 0) -++/* Instead of having one common LRU list in the -++ * BPF_MAP_TYPE_LRU_[PERCPU_]HASH map, use a percpu LRU list -++ * which can scale and perform better. -++ * Note, the LRU nodes (including free nodes) cannot be moved -++ * across different LRU lists. -++ */ -++#define BPF_F_NO_COMMON_LRU (1U << 1) -++/* Specify numa node during map creation */ -++#define BPF_F_NUMA_NODE (1U << 2) -++ -++#define BPF_OBJ_NAME_LEN 16U -++ -++/* Flags for accessing BPF object from syscall side. */ -++#define BPF_F_RDONLY (1U << 3) -++#define BPF_F_WRONLY (1U << 4) -++ -++/* Flag for stack_map, store build_id+offset instead of pointer */ -++#define BPF_F_STACK_BUILD_ID (1U << 5) -++ -++/* Zero-initialize hash function seed. This should only be used for testing. */ -++#define BPF_F_ZERO_SEED (1U << 6) -++ -++/* Flags for accessing BPF object from program side. */ -++#define BPF_F_RDONLY_PROG (1U << 7) -++#define BPF_F_WRONLY_PROG (1U << 8) -++ -++/* Clone map from listener for newly accepted socket */ -++#define BPF_F_CLONE (1U << 9) -++ -++/* flags for BPF_PROG_QUERY */ -++#define BPF_F_QUERY_EFFECTIVE (1U << 0) -++ -++enum bpf_stack_build_id_status { -++ /* user space need an empty entry to identify end of a trace */ -++ BPF_STACK_BUILD_ID_EMPTY = 0, -++ /* with valid build_id and offset */ -++ BPF_STACK_BUILD_ID_VALID = 1, -++ /* couldn't get build_id, fallback to ip */ -++ BPF_STACK_BUILD_ID_IP = 2, -++}; -++ -++#define BPF_BUILD_ID_SIZE 20 -++struct bpf_stack_build_id { -++ __s32 status; -++ unsigned char build_id[BPF_BUILD_ID_SIZE]; -++ union { -++ __u64 offset; -++ __u64 ip; -++ }; -++}; -+ -+ union bpf_attr { -+ struct { /* anonymous struct used by BPF_MAP_CREATE command */ -+@@ -104,6 +372,18 @@ union bpf_attr { -+ __u32 key_size; /* size of key in bytes */ -+ __u32 value_size; /* size of value in bytes */ -+ __u32 max_entries; /* max number of entries in a map */ -++ __u32 map_flags; /* BPF_MAP_CREATE related -++ * flags defined above. -++ */ -++ __u32 inner_map_fd; /* fd pointing to the inner map */ -++ __u32 numa_node; /* numa node (effective only if -++ * BPF_F_NUMA_NODE is set). -++ */ -++ char map_name[BPF_OBJ_NAME_LEN]; -++ __u32 map_ifindex; /* ifindex of netdev to create on */ -++ __u32 btf_fd; /* fd pointing to a BTF type data */ -++ __u32 btf_key_type_id; /* BTF type_id of the key */ -++ __u32 btf_value_type_id; /* BTF type_id of the value */ -+ }; -+ -+ struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */ -+@@ -124,154 +404,2568 @@ union bpf_attr { -+ __u32 log_level; /* verbosity level of verifier */ -+ __u32 log_size; /* size of user buffer */ -+ __aligned_u64 log_buf; /* user supplied buffer */ -+- __u32 kern_version; /* checked when prog_type=kprobe */ -++ __u32 kern_version; /* not used */ -++ __u32 prog_flags; -++ char prog_name[BPF_OBJ_NAME_LEN]; -++ __u32 prog_ifindex; /* ifindex of netdev to prep for */ -++ /* For some prog types expected attach type must be known at -++ * load time to verify attach type specific parts of prog -++ * (context accesses, allowed helpers, etc). -++ */ -++ __u32 expected_attach_type; -++ __u32 prog_btf_fd; /* fd pointing to BTF type data */ -++ __u32 func_info_rec_size; /* userspace bpf_func_info size */ -++ __aligned_u64 func_info; /* func info */ -++ __u32 func_info_cnt; /* number of bpf_func_info records */ -++ __u32 line_info_rec_size; /* userspace bpf_line_info size */ -++ __aligned_u64 line_info; /* line info */ -++ __u32 line_info_cnt; /* number of bpf_line_info records */ -+ }; -+ -+ struct { /* anonymous struct used by BPF_OBJ_* commands */ -+ __aligned_u64 pathname; -+ __u32 bpf_fd; -++ __u32 file_flags; -++ }; -++ -++ struct { /* anonymous struct used by BPF_PROG_ATTACH/DETACH commands */ -++ __u32 target_fd; /* container object to attach to */ -++ __u32 attach_bpf_fd; /* eBPF program to attach */ -++ __u32 attach_type; -++ __u32 attach_flags; -++ }; -++ -++ struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ -++ __u32 prog_fd; -++ __u32 retval; -++ __u32 data_size_in; /* input: len of data_in */ -++ __u32 data_size_out; /* input/output: len of data_out -++ * returns ENOSPC if data_out -++ * is too small. -++ */ -++ __aligned_u64 data_in; -++ __aligned_u64 data_out; -++ __u32 repeat; -++ __u32 duration; -++ __u32 ctx_size_in; /* input: len of ctx_in */ -++ __u32 ctx_size_out; /* input/output: len of ctx_out -++ * returns ENOSPC if ctx_out -++ * is too small. -++ */ -++ __aligned_u64 ctx_in; -++ __aligned_u64 ctx_out; -++ } test; -++ -++ struct { /* anonymous struct used by BPF_*_GET_*_ID */ -++ union { -++ __u32 start_id; -++ __u32 prog_id; -++ __u32 map_id; -++ __u32 btf_id; -++ }; -++ __u32 next_id; -++ __u32 open_flags; -+ }; -++ -++ struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */ -++ __u32 bpf_fd; -++ __u32 info_len; -++ __aligned_u64 info; -++ } info; -++ -++ struct { /* anonymous struct used by BPF_PROG_QUERY command */ -++ __u32 target_fd; /* container object to query */ -++ __u32 attach_type; -++ __u32 query_flags; -++ __u32 attach_flags; -++ __aligned_u64 prog_ids; -++ __u32 prog_cnt; -++ } query; -++ -++ struct { -++ __u64 name; -++ __u32 prog_fd; -++ } raw_tracepoint; -++ -++ struct { /* anonymous struct for BPF_BTF_LOAD */ -++ __aligned_u64 btf; -++ __aligned_u64 btf_log_buf; -++ __u32 btf_size; -++ __u32 btf_log_size; -++ __u32 btf_log_level; -++ }; -++ -++ struct { -++ __u32 pid; /* input: pid */ -++ __u32 fd; /* input: fd */ -++ __u32 flags; /* input: flags */ -++ __u32 buf_len; /* input/output: buf len */ -++ __aligned_u64 buf; /* input/output: -++ * tp_name for tracepoint -++ * symbol for kprobe -++ * filename for uprobe -++ */ -++ __u32 prog_id; /* output: prod_id */ -++ __u32 fd_type; /* output: BPF_FD_TYPE_* */ -++ __u64 probe_offset; /* output: probe_offset */ -++ __u64 probe_addr; /* output: probe_addr */ -++ } task_fd_query; -+ } __attribute__((aligned(8))); -+ -++/* The description below is an attempt at providing documentation to eBPF -++ * developers about the multiple available eBPF helper functions. It can be -++ * parsed and used to produce a manual page. The workflow is the following, -++ * and requires the rst2man utility: -++ * -++ * $ ./scripts/bpf_helpers_doc.py \ -++ * --filename include/uapi/linux/bpf.h > /tmp/bpf-helpers.rst -++ * $ rst2man /tmp/bpf-helpers.rst > /tmp/bpf-helpers.7 -++ * $ man /tmp/bpf-helpers.7 -++ * -++ * Note that in order to produce this external documentation, some RST -++ * formatting is used in the descriptions to get "bold" and "italics" in -++ * manual pages. Also note that the few trailing white spaces are -++ * intentional, removing them would break paragraphs for rst2man. -++ * -++ * Start of BPF helper function descriptions: -++ * -++ * void *bpf_map_lookup_elem(struct bpf_map *map, const void *key) -++ * Description -++ * Perform a lookup in *map* for an entry associated to *key*. -++ * Return -++ * Map value associated to *key*, or **NULL** if no entry was -++ * found. -++ * -++ * int bpf_map_update_elem(struct bpf_map *map, const void *key, const void *value, u64 flags) -++ * Description -++ * Add or update the value of the entry associated to *key* in -++ * *map* with *value*. *flags* is one of: -++ * -++ * **BPF_NOEXIST** -++ * The entry for *key* must not exist in the map. -++ * **BPF_EXIST** -++ * The entry for *key* must already exist in the map. -++ * **BPF_ANY** -++ * No condition on the existence of the entry for *key*. -++ * -++ * Flag value **BPF_NOEXIST** cannot be used for maps of types -++ * **BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY** (all -++ * elements always exist), the helper would return an error. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_map_delete_elem(struct bpf_map *map, const void *key) -++ * Description -++ * Delete entry with *key* from *map*. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_probe_read(void *dst, u32 size, const void *src) -++ * Description -++ * For tracing programs, safely attempt to read *size* bytes from -++ * address *src* and store the data in *dst*. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * u64 bpf_ktime_get_ns(void) -++ * Description -++ * Return the time elapsed since system boot, in nanoseconds. -++ * Return -++ * Current *ktime*. -++ * -++ * int bpf_trace_printk(const char *fmt, u32 fmt_size, ...) -++ * Description -++ * This helper is a "printk()-like" facility for debugging. It -++ * prints a message defined by format *fmt* (of size *fmt_size*) -++ * to file *\/sys/kernel/debug/tracing/trace* from DebugFS, if -++ * available. It can take up to three additional **u64** -++ * arguments (as an eBPF helpers, the total number of arguments is -++ * limited to five). -++ * -++ * Each time the helper is called, it appends a line to the trace. -++ * Lines are discarded while *\/sys/kernel/debug/tracing/trace* is -++ * open, use *\/sys/kernel/debug/tracing/trace_pipe* to avoid this. -++ * The format of the trace is customizable, and the exact output -++ * one will get depends on the options set in -++ * *\/sys/kernel/debug/tracing/trace_options* (see also the -++ * *README* file under the same directory). However, it usually -++ * defaults to something like: -++ * -++ * :: -++ * -++ * telnet-470 [001] .N.. 419421.045894: 0x00000001: -++ * -++ * In the above: -++ * -++ * * ``telnet`` is the name of the current task. -++ * * ``470`` is the PID of the current task. -++ * * ``001`` is the CPU number on which the task is -++ * running. -++ * * In ``.N..``, each character refers to a set of -++ * options (whether irqs are enabled, scheduling -++ * options, whether hard/softirqs are running, level of -++ * preempt_disabled respectively). **N** means that -++ * **TIF_NEED_RESCHED** and **PREEMPT_NEED_RESCHED** -++ * are set. -++ * * ``419421.045894`` is a timestamp. -++ * * ``0x00000001`` is a fake value used by BPF for the -++ * instruction pointer register. -++ * * ```` is the message formatted with -++ * *fmt*. -++ * -++ * The conversion specifiers supported by *fmt* are similar, but -++ * more limited than for printk(). They are **%d**, **%i**, -++ * **%u**, **%x**, **%ld**, **%li**, **%lu**, **%lx**, **%lld**, -++ * **%lli**, **%llu**, **%llx**, **%p**, **%s**. No modifier (size -++ * of field, padding with zeroes, etc.) is available, and the -++ * helper will return **-EINVAL** (but print nothing) if it -++ * encounters an unknown specifier. -++ * -++ * Also, note that **bpf_trace_printk**\ () is slow, and should -++ * only be used for debugging purposes. For this reason, a notice -++ * bloc (spanning several lines) is printed to kernel logs and -++ * states that the helper should not be used "for production use" -++ * the first time this helper is used (or more precisely, when -++ * **trace_printk**\ () buffers are allocated). For passing values -++ * to user space, perf events should be preferred. -++ * Return -++ * The number of bytes written to the buffer, or a negative error -++ * in case of failure. -++ * -++ * u32 bpf_get_prandom_u32(void) -++ * Description -++ * Get a pseudo-random number. -++ * -++ * From a security point of view, this helper uses its own -++ * pseudo-random internal state, and cannot be used to infer the -++ * seed of other random functions in the kernel. However, it is -++ * essential to note that the generator used by the helper is not -++ * cryptographically secure. -++ * Return -++ * A random 32-bit unsigned value. -++ * -++ * u32 bpf_get_smp_processor_id(void) -++ * Description -++ * Get the SMP (symmetric multiprocessing) processor id. Note that -++ * all programs run with preemption disabled, which means that the -++ * SMP processor id is stable during all the execution of the -++ * program. -++ * Return -++ * The SMP id of the processor running the program. -++ * -++ * int bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags) -++ * Description -++ * Store *len* bytes from address *from* into the packet -++ * associated to *skb*, at *offset*. *flags* are a combination of -++ * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the -++ * checksum for the packet after storing the bytes) and -++ * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\ -++ * **->swhash** and *skb*\ **->l4hash** to 0). -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_l3_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 size) -++ * Description -++ * Recompute the layer 3 (e.g. IP) checksum for the packet -++ * associated to *skb*. Computation is incremental, so the helper -++ * must know the former value of the header field that was -++ * modified (*from*), the new value of this field (*to*), and the -++ * number of bytes (2 or 4) for this field, stored in *size*. -++ * Alternatively, it is possible to store the difference between -++ * the previous and the new values of the header field in *to*, by -++ * setting *from* and *size* to 0. For both methods, *offset* -++ * indicates the location of the IP checksum within the packet. -++ * -++ * This helper works in combination with **bpf_csum_diff**\ (), -++ * which does not update the checksum in-place, but offers more -++ * flexibility and can handle sizes larger than 2 or 4 for the -++ * checksum to update. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_l4_csum_replace(struct sk_buff *skb, u32 offset, u64 from, u64 to, u64 flags) -++ * Description -++ * Recompute the layer 4 (e.g. TCP, UDP or ICMP) checksum for the -++ * packet associated to *skb*. Computation is incremental, so the -++ * helper must know the former value of the header field that was -++ * modified (*from*), the new value of this field (*to*), and the -++ * number of bytes (2 or 4) for this field, stored on the lowest -++ * four bits of *flags*. Alternatively, it is possible to store -++ * the difference between the previous and the new values of the -++ * header field in *to*, by setting *from* and the four lowest -++ * bits of *flags* to 0. For both methods, *offset* indicates the -++ * location of the IP checksum within the packet. In addition to -++ * the size of the field, *flags* can be added (bitwise OR) actual -++ * flags. With **BPF_F_MARK_MANGLED_0**, a null checksum is left -++ * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and -++ * for updates resulting in a null checksum the value is set to -++ * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates -++ * the checksum is to be computed against a pseudo-header. -++ * -++ * This helper works in combination with **bpf_csum_diff**\ (), -++ * which does not update the checksum in-place, but offers more -++ * flexibility and can handle sizes larger than 2 or 4 for the -++ * checksum to update. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_tail_call(void *ctx, struct bpf_map *prog_array_map, u32 index) -++ * Description -++ * This special helper is used to trigger a "tail call", or in -++ * other words, to jump into another eBPF program. The same stack -++ * frame is used (but values on stack and in registers for the -++ * caller are not accessible to the callee). This mechanism allows -++ * for program chaining, either for raising the maximum number of -++ * available eBPF instructions, or to execute given programs in -++ * conditional blocks. For security reasons, there is an upper -++ * limit to the number of successive tail calls that can be -++ * performed. -++ * -++ * Upon call of this helper, the program attempts to jump into a -++ * program referenced at index *index* in *prog_array_map*, a -++ * special map of type **BPF_MAP_TYPE_PROG_ARRAY**, and passes -++ * *ctx*, a pointer to the context. -++ * -++ * If the call succeeds, the kernel immediately runs the first -++ * instruction of the new program. This is not a function call, -++ * and it never returns to the previous program. If the call -++ * fails, then the helper has no effect, and the caller continues -++ * to run its subsequent instructions. A call can fail if the -++ * destination program for the jump does not exist (i.e. *index* -++ * is superior to the number of entries in *prog_array_map*), or -++ * if the maximum number of tail calls has been reached for this -++ * chain of programs. This limit is defined in the kernel by the -++ * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), -++ * which is currently set to 32. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_clone_redirect(struct sk_buff *skb, u32 ifindex, u64 flags) -++ * Description -++ * Clone and redirect the packet associated to *skb* to another -++ * net device of index *ifindex*. Both ingress and egress -++ * interfaces can be used for redirection. The **BPF_F_INGRESS** -++ * value in *flags* is used to make the distinction (ingress path -++ * is selected if the flag is present, egress path otherwise). -++ * This is the only flag supported for now. -++ * -++ * In comparison with **bpf_redirect**\ () helper, -++ * **bpf_clone_redirect**\ () has the associated cost of -++ * duplicating the packet buffer, but this can be executed out of -++ * the eBPF program. Conversely, **bpf_redirect**\ () is more -++ * efficient, but it is handled through an action code where the -++ * redirection happens only after the eBPF program has returned. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * u64 bpf_get_current_pid_tgid(void) -++ * Return -++ * A 64-bit integer containing the current tgid and pid, and -++ * created as such: -++ * *current_task*\ **->tgid << 32 \|** -++ * *current_task*\ **->pid**. -++ * -++ * u64 bpf_get_current_uid_gid(void) -++ * Return -++ * A 64-bit integer containing the current GID and UID, and -++ * created as such: *current_gid* **<< 32 \|** *current_uid*. -++ * -++ * int bpf_get_current_comm(char *buf, u32 size_of_buf) -++ * Description -++ * Copy the **comm** attribute of the current task into *buf* of -++ * *size_of_buf*. The **comm** attribute contains the name of -++ * the executable (excluding the path) for the current task. The -++ * *size_of_buf* must be strictly positive. On success, the -++ * helper makes sure that the *buf* is NUL-terminated. On failure, -++ * it is filled with zeroes. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * u32 bpf_get_cgroup_classid(struct sk_buff *skb) -++ * Description -++ * Retrieve the classid for the current task, i.e. for the net_cls -++ * cgroup to which *skb* belongs. -++ * -++ * This helper can be used on TC egress path, but not on ingress. -++ * -++ * The net_cls cgroup provides an interface to tag network packets -++ * based on a user-provided identifier for all traffic coming from -++ * the tasks belonging to the related cgroup. See also the related -++ * kernel documentation, available from the Linux sources in file -++ * *Documentation/admin-guide/cgroup-v1/net_cls.rst*. -++ * -++ * The Linux kernel has two versions for cgroups: there are -++ * cgroups v1 and cgroups v2. Both are available to users, who can -++ * use a mixture of them, but note that the net_cls cgroup is for -++ * cgroup v1 only. This makes it incompatible with BPF programs -++ * run on cgroups, which is a cgroup-v2-only feature (a socket can -++ * only hold data for one version of cgroups at a time). -++ * -++ * This helper is only available is the kernel was compiled with -++ * the **CONFIG_CGROUP_NET_CLASSID** configuration option set to -++ * "**y**" or to "**m**". -++ * Return -++ * The classid, or 0 for the default unconfigured classid. -++ * -++ * int bpf_skb_vlan_push(struct sk_buff *skb, __be16 vlan_proto, u16 vlan_tci) -++ * Description -++ * Push a *vlan_tci* (VLAN tag control information) of protocol -++ * *vlan_proto* to the packet associated to *skb*, then update -++ * the checksum. Note that if *vlan_proto* is different from -++ * **ETH_P_8021Q** and **ETH_P_8021AD**, it is considered to -++ * be **ETH_P_8021Q**. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_vlan_pop(struct sk_buff *skb) -++ * Description -++ * Pop a VLAN header from the packet associated to *skb*. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_get_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) -++ * Description -++ * Get tunnel metadata. This helper takes a pointer *key* to an -++ * empty **struct bpf_tunnel_key** of **size**, that will be -++ * filled with tunnel metadata for the packet associated to *skb*. -++ * The *flags* can be set to **BPF_F_TUNINFO_IPV6**, which -++ * indicates that the tunnel is based on IPv6 protocol instead of -++ * IPv4. -++ * -++ * The **struct bpf_tunnel_key** is an object that generalizes the -++ * principal parameters used by various tunneling protocols into a -++ * single struct. This way, it can be used to easily make a -++ * decision based on the contents of the encapsulation header, -++ * "summarized" in this struct. In particular, it holds the IP -++ * address of the remote end (IPv4 or IPv6, depending on the case) -++ * in *key*\ **->remote_ipv4** or *key*\ **->remote_ipv6**. Also, -++ * this struct exposes the *key*\ **->tunnel_id**, which is -++ * generally mapped to a VNI (Virtual Network Identifier), making -++ * it programmable together with the **bpf_skb_set_tunnel_key**\ -++ * () helper. -++ * -++ * Let's imagine that the following code is part of a program -++ * attached to the TC ingress interface, on one end of a GRE -++ * tunnel, and is supposed to filter out all messages coming from -++ * remote ends with IPv4 address other than 10.0.0.1: -++ * -++ * :: -++ * -++ * int ret; -++ * struct bpf_tunnel_key key = {}; -++ * -++ * ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); -++ * if (ret < 0) -++ * return TC_ACT_SHOT; // drop packet -++ * -++ * if (key.remote_ipv4 != 0x0a000001) -++ * return TC_ACT_SHOT; // drop packet -++ * -++ * return TC_ACT_OK; // accept packet -++ * -++ * This interface can also be used with all encapsulation devices -++ * that can operate in "collect metadata" mode: instead of having -++ * one network device per specific configuration, the "collect -++ * metadata" mode only requires a single device where the -++ * configuration can be extracted from this helper. -++ * -++ * This can be used together with various tunnels such as VXLan, -++ * Geneve, GRE or IP in IP (IPIP). -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_set_tunnel_key(struct sk_buff *skb, struct bpf_tunnel_key *key, u32 size, u64 flags) -++ * Description -++ * Populate tunnel metadata for packet associated to *skb.* The -++ * tunnel metadata is set to the contents of *key*, of *size*. The -++ * *flags* can be set to a combination of the following values: -++ * -++ * **BPF_F_TUNINFO_IPV6** -++ * Indicate that the tunnel is based on IPv6 protocol -++ * instead of IPv4. -++ * **BPF_F_ZERO_CSUM_TX** -++ * For IPv4 packets, add a flag to tunnel metadata -++ * indicating that checksum computation should be skipped -++ * and checksum set to zeroes. -++ * **BPF_F_DONT_FRAGMENT** -++ * Add a flag to tunnel metadata indicating that the -++ * packet should not be fragmented. -++ * **BPF_F_SEQ_NUMBER** -++ * Add a flag to tunnel metadata indicating that a -++ * sequence number should be added to tunnel header before -++ * sending the packet. This flag was added for GRE -++ * encapsulation, but might be used with other protocols -++ * as well in the future. -++ * -++ * Here is a typical usage on the transmit path: -++ * -++ * :: -++ * -++ * struct bpf_tunnel_key key; -++ * populate key ... -++ * bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0); -++ * bpf_clone_redirect(skb, vxlan_dev_ifindex, 0); -++ * -++ * See also the description of the **bpf_skb_get_tunnel_key**\ () -++ * helper for additional information. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * u64 bpf_perf_event_read(struct bpf_map *map, u64 flags) -++ * Description -++ * Read the value of a perf event counter. This helper relies on a -++ * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of -++ * the perf event counter is selected when *map* is updated with -++ * perf event file descriptors. The *map* is an array whose size -++ * is the number of available CPUs, and each cell contains a value -++ * relative to one CPU. The value to retrieve is indicated by -++ * *flags*, that contains the index of the CPU to look up, masked -++ * with **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to -++ * **BPF_F_CURRENT_CPU** to indicate that the value for the -++ * current CPU should be retrieved. -++ * -++ * Note that before Linux 4.13, only hardware perf event can be -++ * retrieved. -++ * -++ * Also, be aware that the newer helper -++ * **bpf_perf_event_read_value**\ () is recommended over -++ * **bpf_perf_event_read**\ () in general. The latter has some ABI -++ * quirks where error and counter value are used as a return code -++ * (which is wrong to do since ranges may overlap). This issue is -++ * fixed with **bpf_perf_event_read_value**\ (), which at the same -++ * time provides more features over the **bpf_perf_event_read**\ -++ * () interface. Please refer to the description of -++ * **bpf_perf_event_read_value**\ () for details. -++ * Return -++ * The value of the perf event counter read from the map, or a -++ * negative error code in case of failure. -++ * -++ * int bpf_redirect(u32 ifindex, u64 flags) -++ * Description -++ * Redirect the packet to another net device of index *ifindex*. -++ * This helper is somewhat similar to **bpf_clone_redirect**\ -++ * (), except that the packet is not cloned, which provides -++ * increased performance. -++ * -++ * Except for XDP, both ingress and egress interfaces can be used -++ * for redirection. The **BPF_F_INGRESS** value in *flags* is used -++ * to make the distinction (ingress path is selected if the flag -++ * is present, egress path otherwise). Currently, XDP only -++ * supports redirection to the egress interface, and accepts no -++ * flag at all. -++ * -++ * The same effect can be attained with the more generic -++ * **bpf_redirect_map**\ (), which requires specific maps to be -++ * used but offers better performance. -++ * Return -++ * For XDP, the helper returns **XDP_REDIRECT** on success or -++ * **XDP_ABORTED** on error. For other program types, the values -++ * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on -++ * error. -++ * -++ * u32 bpf_get_route_realm(struct sk_buff *skb) -++ * Description -++ * Retrieve the realm or the route, that is to say the -++ * **tclassid** field of the destination for the *skb*. The -++ * indentifier retrieved is a user-provided tag, similar to the -++ * one used with the net_cls cgroup (see description for -++ * **bpf_get_cgroup_classid**\ () helper), but here this tag is -++ * held by a route (a destination entry), not by a task. -++ * -++ * Retrieving this identifier works with the clsact TC egress hook -++ * (see also **tc-bpf(8)**), or alternatively on conventional -++ * classful egress qdiscs, but not on TC ingress path. In case of -++ * clsact TC egress hook, this has the advantage that, internally, -++ * the destination entry has not been dropped yet in the transmit -++ * path. Therefore, the destination entry does not need to be -++ * artificially held via **netif_keep_dst**\ () for a classful -++ * qdisc until the *skb* is freed. -++ * -++ * This helper is available only if the kernel was compiled with -++ * **CONFIG_IP_ROUTE_CLASSID** configuration option. -++ * Return -++ * The realm of the route for the packet associated to *skb*, or 0 -++ * if none was found. -++ * -++ * int bpf_perf_event_output(struct pt_regs *ctx, struct bpf_map *map, u64 flags, void *data, u64 size) -++ * Description -++ * Write raw *data* blob into a special BPF perf event held by -++ * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf -++ * event must have the following attributes: **PERF_SAMPLE_RAW** -++ * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and -++ * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. -++ * -++ * The *flags* are used to indicate the index in *map* for which -++ * the value must be put, masked with **BPF_F_INDEX_MASK**. -++ * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** -++ * to indicate that the index of the current CPU core should be -++ * used. -++ * -++ * The value to write, of *size*, is passed through eBPF stack and -++ * pointed by *data*. -++ * -++ * The context of the program *ctx* needs also be passed to the -++ * helper. -++ * -++ * On user space, a program willing to read the values needs to -++ * call **perf_event_open**\ () on the perf event (either for -++ * one or for all CPUs) and to store the file descriptor into the -++ * *map*. This must be done before the eBPF program can send data -++ * into it. An example is available in file -++ * *samples/bpf/trace_output_user.c* in the Linux kernel source -++ * tree (the eBPF program counterpart is in -++ * *samples/bpf/trace_output_kern.c*). -++ * -++ * **bpf_perf_event_output**\ () achieves better performance -++ * than **bpf_trace_printk**\ () for sharing data with user -++ * space, and is much better suitable for streaming data from eBPF -++ * programs. -++ * -++ * Note that this helper is not restricted to tracing use cases -++ * and can be used with programs attached to TC or XDP as well, -++ * where it allows for passing data to user space listeners. Data -++ * can be: -++ * -++ * * Only custom structs, -++ * * Only the packet payload, or -++ * * A combination of both. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len) -++ * Description -++ * This helper was provided as an easy way to load data from a -++ * packet. It can be used to load *len* bytes from *offset* from -++ * the packet associated to *skb*, into the buffer pointed by -++ * *to*. -++ * -++ * Since Linux 4.7, usage of this helper has mostly been replaced -++ * by "direct packet access", enabling packet data to be -++ * manipulated with *skb*\ **->data** and *skb*\ **->data_end** -++ * pointing respectively to the first byte of packet data and to -++ * the byte after the last byte of packet data. However, it -++ * remains useful if one wishes to read large quantities of data -++ * at once from a packet into the eBPF stack. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_get_stackid(struct pt_regs *ctx, struct bpf_map *map, u64 flags) -++ * Description -++ * Walk a user or a kernel stack and return its id. To achieve -++ * this, the helper needs *ctx*, which is a pointer to the context -++ * on which the tracing program is executed, and a pointer to a -++ * *map* of type **BPF_MAP_TYPE_STACK_TRACE**. -++ * -++ * The last argument, *flags*, holds the number of stack frames to -++ * skip (from 0 to 255), masked with -++ * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set -++ * a combination of the following flags: -++ * -++ * **BPF_F_USER_STACK** -++ * Collect a user space stack instead of a kernel stack. -++ * **BPF_F_FAST_STACK_CMP** -++ * Compare stacks by hash only. -++ * **BPF_F_REUSE_STACKID** -++ * If two different stacks hash into the same *stackid*, -++ * discard the old one. -++ * -++ * The stack id retrieved is a 32 bit long integer handle which -++ * can be further combined with other data (including other stack -++ * ids) and used as a key into maps. This can be useful for -++ * generating a variety of graphs (such as flame graphs or off-cpu -++ * graphs). -++ * -++ * For walking a stack, this helper is an improvement over -++ * **bpf_probe_read**\ (), which can be used with unrolled loops -++ * but is not efficient and consumes a lot of eBPF instructions. -++ * Instead, **bpf_get_stackid**\ () can collect up to -++ * **PERF_MAX_STACK_DEPTH** both kernel and user frames. Note that -++ * this limit can be controlled with the **sysctl** program, and -++ * that it should be manually increased in order to profile long -++ * user stacks (such as stacks for Java programs). To do so, use: -++ * -++ * :: -++ * -++ * # sysctl kernel.perf_event_max_stack= -++ * Return -++ * The positive or null stack id on success, or a negative error -++ * in case of failure. -++ * -++ * s64 bpf_csum_diff(__be32 *from, u32 from_size, __be32 *to, u32 to_size, __wsum seed) -++ * Description -++ * Compute a checksum difference, from the raw buffer pointed by -++ * *from*, of length *from_size* (that must be a multiple of 4), -++ * towards the raw buffer pointed by *to*, of size *to_size* -++ * (same remark). An optional *seed* can be added to the value -++ * (this can be cascaded, the seed may come from a previous call -++ * to the helper). -++ * -++ * This is flexible enough to be used in several ways: -++ * -++ * * With *from_size* == 0, *to_size* > 0 and *seed* set to -++ * checksum, it can be used when pushing new data. -++ * * With *from_size* > 0, *to_size* == 0 and *seed* set to -++ * checksum, it can be used when removing data from a packet. -++ * * With *from_size* > 0, *to_size* > 0 and *seed* set to 0, it -++ * can be used to compute a diff. Note that *from_size* and -++ * *to_size* do not need to be equal. -++ * -++ * This helper can be used in combination with -++ * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ (), to -++ * which one can feed in the difference computed with -++ * **bpf_csum_diff**\ (). -++ * Return -++ * The checksum result, or a negative error code in case of -++ * failure. -++ * -++ * int bpf_skb_get_tunnel_opt(struct sk_buff *skb, u8 *opt, u32 size) -++ * Description -++ * Retrieve tunnel options metadata for the packet associated to -++ * *skb*, and store the raw tunnel option data to the buffer *opt* -++ * of *size*. -++ * -++ * This helper can be used with encapsulation devices that can -++ * operate in "collect metadata" mode (please refer to the related -++ * note in the description of **bpf_skb_get_tunnel_key**\ () for -++ * more details). A particular example where this can be used is -++ * in combination with the Geneve encapsulation protocol, where it -++ * allows for pushing (with **bpf_skb_get_tunnel_opt**\ () helper) -++ * and retrieving arbitrary TLVs (Type-Length-Value headers) from -++ * the eBPF program. This allows for full customization of these -++ * headers. -++ * Return -++ * The size of the option data retrieved. -++ * -++ * int bpf_skb_set_tunnel_opt(struct sk_buff *skb, u8 *opt, u32 size) -++ * Description -++ * Set tunnel options metadata for the packet associated to *skb* -++ * to the option data contained in the raw buffer *opt* of *size*. -++ * -++ * See also the description of the **bpf_skb_get_tunnel_opt**\ () -++ * helper for additional information. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_change_proto(struct sk_buff *skb, __be16 proto, u64 flags) -++ * Description -++ * Change the protocol of the *skb* to *proto*. Currently -++ * supported are transition from IPv4 to IPv6, and from IPv6 to -++ * IPv4. The helper takes care of the groundwork for the -++ * transition, including resizing the socket buffer. The eBPF -++ * program is expected to fill the new headers, if any, via -++ * **skb_store_bytes**\ () and to recompute the checksums with -++ * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ -++ * (). The main case for this helper is to perform NAT64 -++ * operations out of an eBPF program. -++ * -++ * Internally, the GSO type is marked as dodgy so that headers are -++ * checked and segments are recalculated by the GSO/GRO engine. -++ * The size for GSO target is adapted as well. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_change_type(struct sk_buff *skb, u32 type) -++ * Description -++ * Change the packet type for the packet associated to *skb*. This -++ * comes down to setting *skb*\ **->pkt_type** to *type*, except -++ * the eBPF program does not have a write access to *skb*\ -++ * **->pkt_type** beside this helper. Using a helper here allows -++ * for graceful handling of errors. -++ * -++ * The major use case is to change incoming *skb*s to -++ * **PACKET_HOST** in a programmatic way instead of having to -++ * recirculate via **redirect**\ (..., **BPF_F_INGRESS**), for -++ * example. -++ * -++ * Note that *type* only allows certain values. At this time, they -++ * are: -++ * -++ * **PACKET_HOST** -++ * Packet is for us. -++ * **PACKET_BROADCAST** -++ * Send packet to all. -++ * **PACKET_MULTICAST** -++ * Send packet to group. -++ * **PACKET_OTHERHOST** -++ * Send packet to someone else. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_under_cgroup(struct sk_buff *skb, struct bpf_map *map, u32 index) -++ * Description -++ * Check whether *skb* is a descendant of the cgroup2 held by -++ * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. -++ * Return -++ * The return value depends on the result of the test, and can be: -++ * -++ * * 0, if the *skb* failed the cgroup2 descendant test. -++ * * 1, if the *skb* succeeded the cgroup2 descendant test. -++ * * A negative error code, if an error occurred. -++ * -++ * u32 bpf_get_hash_recalc(struct sk_buff *skb) -++ * Description -++ * Retrieve the hash of the packet, *skb*\ **->hash**. If it is -++ * not set, in particular if the hash was cleared due to mangling, -++ * recompute this hash. Later accesses to the hash can be done -++ * directly with *skb*\ **->hash**. -++ * -++ * Calling **bpf_set_hash_invalid**\ (), changing a packet -++ * prototype with **bpf_skb_change_proto**\ (), or calling -++ * **bpf_skb_store_bytes**\ () with the -++ * **BPF_F_INVALIDATE_HASH** are actions susceptible to clear -++ * the hash and to trigger a new computation for the next call to -++ * **bpf_get_hash_recalc**\ (). -++ * Return -++ * The 32-bit hash. -++ * -++ * u64 bpf_get_current_task(void) -++ * Return -++ * A pointer to the current task struct. -++ * -++ * int bpf_probe_write_user(void *dst, const void *src, u32 len) -++ * Description -++ * Attempt in a safe way to write *len* bytes from the buffer -++ * *src* to *dst* in memory. It only works for threads that are in -++ * user context, and *dst* must be a valid user space address. -++ * -++ * This helper should not be used to implement any kind of -++ * security mechanism because of TOC-TOU attacks, but rather to -++ * debug, divert, and manipulate execution of semi-cooperative -++ * processes. -++ * -++ * Keep in mind that this feature is meant for experiments, and it -++ * has a risk of crashing the system and running programs. -++ * Therefore, when an eBPF program using this helper is attached, -++ * a warning including PID and process name is printed to kernel -++ * logs. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_current_task_under_cgroup(struct bpf_map *map, u32 index) -++ * Description -++ * Check whether the probe is being run is the context of a given -++ * subset of the cgroup2 hierarchy. The cgroup2 to test is held by -++ * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. -++ * Return -++ * The return value depends on the result of the test, and can be: -++ * -++ * * 0, if current task belongs to the cgroup2. -++ * * 1, if current task does not belong to the cgroup2. -++ * * A negative error code, if an error occurred. -++ * -++ * int bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags) -++ * Description -++ * Resize (trim or grow) the packet associated to *skb* to the -++ * new *len*. The *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * The basic idea is that the helper performs the needed work to -++ * change the size of the packet, then the eBPF program rewrites -++ * the rest via helpers like **bpf_skb_store_bytes**\ (), -++ * **bpf_l3_csum_replace**\ (), **bpf_l3_csum_replace**\ () -++ * and others. This helper is a slow path utility intended for -++ * replies with control messages. And because it is targeted for -++ * slow path, the helper itself can afford to be slow: it -++ * implicitly linearizes, unclones and drops offloads from the -++ * *skb*. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_pull_data(struct sk_buff *skb, u32 len) -++ * Description -++ * Pull in non-linear data in case the *skb* is non-linear and not -++ * all of *len* are part of the linear section. Make *len* bytes -++ * from *skb* readable and writable. If a zero value is passed for -++ * *len*, then the whole length of the *skb* is pulled. -++ * -++ * This helper is only needed for reading and writing with direct -++ * packet access. -++ * -++ * For direct packet access, testing that offsets to access -++ * are within packet boundaries (test on *skb*\ **->data_end**) is -++ * susceptible to fail if offsets are invalid, or if the requested -++ * data is in non-linear parts of the *skb*. On failure the -++ * program can just bail out, or in the case of a non-linear -++ * buffer, use a helper to make the data available. The -++ * **bpf_skb_load_bytes**\ () helper is a first solution to access -++ * the data. Another one consists in using **bpf_skb_pull_data** -++ * to pull in once the non-linear parts, then retesting and -++ * eventually access the data. -++ * -++ * At the same time, this also makes sure the *skb* is uncloned, -++ * which is a necessary condition for direct write. As this needs -++ * to be an invariant for the write part only, the verifier -++ * detects writes and adds a prologue that is calling -++ * **bpf_skb_pull_data()** to effectively unclone the *skb* from -++ * the very beginning in case it is indeed cloned. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * s64 bpf_csum_update(struct sk_buff *skb, __wsum csum) -++ * Description -++ * Add the checksum *csum* into *skb*\ **->csum** in case the -++ * driver has supplied a checksum for the entire packet into that -++ * field. Return an error otherwise. This helper is intended to be -++ * used in combination with **bpf_csum_diff**\ (), in particular -++ * when the checksum needs to be updated after data has been -++ * written into the packet through direct packet access. -++ * Return -++ * The checksum on success, or a negative error code in case of -++ * failure. -++ * -++ * void bpf_set_hash_invalid(struct sk_buff *skb) -++ * Description -++ * Invalidate the current *skb*\ **->hash**. It can be used after -++ * mangling on headers through direct packet access, in order to -++ * indicate that the hash is outdated and to trigger a -++ * recalculation the next time the kernel tries to access this -++ * hash or when the **bpf_get_hash_recalc**\ () helper is called. -++ * -++ * int bpf_get_numa_node_id(void) -++ * Description -++ * Return the id of the current NUMA node. The primary use case -++ * for this helper is the selection of sockets for the local NUMA -++ * node, when the program is attached to sockets using the -++ * **SO_ATTACH_REUSEPORT_EBPF** option (see also **socket(7)**), -++ * but the helper is also available to other eBPF program types, -++ * similarly to **bpf_get_smp_processor_id**\ (). -++ * Return -++ * The id of current NUMA node. -++ * -++ * int bpf_skb_change_head(struct sk_buff *skb, u32 len, u64 flags) -++ * Description -++ * Grows headroom of packet associated to *skb* and adjusts the -++ * offset of the MAC header accordingly, adding *len* bytes of -++ * space. It automatically extends and reallocates memory as -++ * required. -++ * -++ * This helper can be used on a layer 3 *skb* to push a MAC header -++ * for redirection into a layer 2 device. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta) -++ * Description -++ * Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that -++ * it is possible to use a negative value for *delta*. This helper -++ * can be used to prepare the packet for pushing or popping -++ * headers. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_probe_read_str(void *dst, int size, const void *unsafe_ptr) -++ * Description -++ * Copy a NUL terminated string from an unsafe address -++ * *unsafe_ptr* to *dst*. The *size* should include the -++ * terminating NUL byte. In case the string length is smaller than -++ * *size*, the target is not padded with further NUL bytes. If the -++ * string length is larger than *size*, just *size*-1 bytes are -++ * copied and the last byte is set to NUL. -++ * -++ * On success, the length of the copied string is returned. This -++ * makes this helper useful in tracing programs for reading -++ * strings, and more importantly to get its length at runtime. See -++ * the following snippet: -++ * -++ * :: -++ * -++ * SEC("kprobe/sys_open") -++ * void bpf_sys_open(struct pt_regs *ctx) -++ * { -++ * char buf[PATHLEN]; // PATHLEN is defined to 256 -++ * int res = bpf_probe_read_str(buf, sizeof(buf), -++ * ctx->di); -++ * -++ * // Consume buf, for example push it to -++ * // userspace via bpf_perf_event_output(); we -++ * // can use res (the string length) as event -++ * // size, after checking its boundaries. -++ * } -++ * -++ * In comparison, using **bpf_probe_read()** helper here instead -++ * to read the string would require to estimate the length at -++ * compile time, and would often result in copying more memory -++ * than necessary. -++ * -++ * Another useful use case is when parsing individual process -++ * arguments or individual environment variables navigating -++ * *current*\ **->mm->arg_start** and *current*\ -++ * **->mm->env_start**: using this helper and the return value, -++ * one can quickly iterate at the right offset of the memory area. -++ * Return -++ * On success, the strictly positive length of the string, -++ * including the trailing NUL character. On error, a negative -++ * value. -++ * -++ * u64 bpf_get_socket_cookie(struct sk_buff *skb) -++ * Description -++ * If the **struct sk_buff** pointed by *skb* has a known socket, -++ * retrieve the cookie (generated by the kernel) of this socket. -++ * If no cookie has been set yet, generate a new cookie. Once -++ * generated, the socket cookie remains stable for the life of the -++ * socket. This helper can be useful for monitoring per socket -++ * networking traffic statistics as it provides a global socket -++ * identifier that can be assumed unique. -++ * Return -++ * A 8-byte long non-decreasing number on success, or 0 if the -++ * socket field is missing inside *skb*. -++ * -++ * u64 bpf_get_socket_cookie(struct bpf_sock_addr *ctx) -++ * Description -++ * Equivalent to bpf_get_socket_cookie() helper that accepts -++ * *skb*, but gets socket from **struct bpf_sock_addr** context. -++ * Return -++ * A 8-byte long non-decreasing number. -++ * -++ * u64 bpf_get_socket_cookie(struct bpf_sock_ops *ctx) -++ * Description -++ * Equivalent to bpf_get_socket_cookie() helper that accepts -++ * *skb*, but gets socket from **struct bpf_sock_ops** context. -++ * Return -++ * A 8-byte long non-decreasing number. -++ * -++ * u32 bpf_get_socket_uid(struct sk_buff *skb) -++ * Return -++ * The owner UID of the socket associated to *skb*. If the socket -++ * is **NULL**, or if it is not a full socket (i.e. if it is a -++ * time-wait or a request socket instead), **overflowuid** value -++ * is returned (note that **overflowuid** might also be the actual -++ * UID value for the socket). -++ * -++ * u32 bpf_set_hash(struct sk_buff *skb, u32 hash) -++ * Description -++ * Set the full hash for *skb* (set the field *skb*\ **->hash**) -++ * to value *hash*. -++ * Return -++ * 0 -++ * -++ * int bpf_setsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen) -++ * Description -++ * Emulate a call to **setsockopt()** on the socket associated to -++ * *bpf_socket*, which must be a full socket. The *level* at -++ * which the option resides and the name *optname* of the option -++ * must be specified, see **setsockopt(2)** for more information. -++ * The option value of length *optlen* is pointed by *optval*. -++ * -++ * This helper actually implements a subset of **setsockopt()**. -++ * It supports the following *level*\ s: -++ * -++ * * **SOL_SOCKET**, which supports the following *optname*\ s: -++ * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, -++ * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**. -++ * * **IPPROTO_TCP**, which supports the following *optname*\ s: -++ * **TCP_CONGESTION**, **TCP_BPF_IW**, -++ * **TCP_BPF_SNDCWND_CLAMP**. -++ * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. -++ * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_adjust_room(struct sk_buff *skb, s32 len_diff, u32 mode, u64 flags) -++ * Description -++ * Grow or shrink the room for data in the packet associated to -++ * *skb* by *len_diff*, and according to the selected *mode*. -++ * -++ * There are two supported modes at this time: -++ * -++ * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer -++ * (room space is added or removed below the layer 2 header). -++ * -++ * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer -++ * (room space is added or removed below the layer 3 header). -++ * -++ * The following flags are supported at this time: -++ * -++ * * **BPF_F_ADJ_ROOM_FIXED_GSO**: Do not adjust gso_size. -++ * Adjusting mss in this way is not allowed for datagrams. -++ * -++ * * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV4**, -++ * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV6**: -++ * Any new space is reserved to hold a tunnel header. -++ * Configure skb offsets and other fields accordingly. -++ * -++ * * **BPF_F_ADJ_ROOM_ENCAP_L4_GRE**, -++ * **BPF_F_ADJ_ROOM_ENCAP_L4_UDP**: -++ * Use with ENCAP_L3 flags to further specify the tunnel type. -++ * -++ * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): -++ * Use with ENCAP_L3/L4 flags to further specify the tunnel -++ * type; *len* is the length of the inner MAC header. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_redirect_map(struct bpf_map *map, u32 key, u64 flags) -++ * Description -++ * Redirect the packet to the endpoint referenced by *map* at -++ * index *key*. Depending on its type, this *map* can contain -++ * references to net devices (for forwarding packets through other -++ * ports), or to CPUs (for redirecting XDP frames to another CPU; -++ * but this is only implemented for native XDP (with driver -++ * support) as of this writing). -++ * -++ * The lower two bits of *flags* are used as the return code if -++ * the map lookup fails. This is so that the return value can be -++ * one of the XDP program return codes up to XDP_TX, as chosen by -++ * the caller. Any higher bits in the *flags* argument must be -++ * unset. -++ * -++ * When used to redirect packets to net devices, this helper -++ * provides a high performance increase over **bpf_redirect**\ (). -++ * This is due to various implementation details of the underlying -++ * mechanisms, one of which is the fact that **bpf_redirect_map**\ -++ * () tries to send packet as a "bulk" to the device. -++ * Return -++ * **XDP_REDIRECT** on success, or **XDP_ABORTED** on error. -++ * -++ * int bpf_sk_redirect_map(struct bpf_map *map, u32 key, u64 flags) -++ * Description -++ * Redirect the packet to the socket referenced by *map* (of type -++ * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and -++ * egress interfaces can be used for redirection. The -++ * **BPF_F_INGRESS** value in *flags* is used to make the -++ * distinction (ingress path is selected if the flag is present, -++ * egress path otherwise). This is the only flag supported for now. -++ * Return -++ * **SK_PASS** on success, or **SK_DROP** on error. -++ * -++ * int bpf_sock_map_update(struct bpf_sock_ops *skops, struct bpf_map *map, void *key, u64 flags) -++ * Description -++ * Add an entry to, or update a *map* referencing sockets. The -++ * *skops* is used as a new value for the entry associated to -++ * *key*. *flags* is one of: -++ * -++ * **BPF_NOEXIST** -++ * The entry for *key* must not exist in the map. -++ * **BPF_EXIST** -++ * The entry for *key* must already exist in the map. -++ * **BPF_ANY** -++ * No condition on the existence of the entry for *key*. -++ * -++ * If the *map* has eBPF programs (parser and verdict), those will -++ * be inherited by the socket being added. If the socket is -++ * already attached to eBPF programs, this results in an error. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta) -++ * Description -++ * Adjust the address pointed by *xdp_md*\ **->data_meta** by -++ * *delta* (which can be positive or negative). Note that this -++ * operation modifies the address stored in *xdp_md*\ **->data**, -++ * so the latter must be loaded only after the helper has been -++ * called. -++ * -++ * The use of *xdp_md*\ **->data_meta** is optional and programs -++ * are not required to use it. The rationale is that when the -++ * packet is processed with XDP (e.g. as DoS filter), it is -++ * possible to push further meta data along with it before passing -++ * to the stack, and to give the guarantee that an ingress eBPF -++ * program attached as a TC classifier on the same device can pick -++ * this up for further post-processing. Since TC works with socket -++ * buffers, it remains possible to set from XDP the **mark** or -++ * **priority** pointers, or other pointers for the socket buffer. -++ * Having this scratch space generic and programmable allows for -++ * more flexibility as the user is free to store whatever meta -++ * data they need. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_perf_event_read_value(struct bpf_map *map, u64 flags, struct bpf_perf_event_value *buf, u32 buf_size) -++ * Description -++ * Read the value of a perf event counter, and store it into *buf* -++ * of size *buf_size*. This helper relies on a *map* of type -++ * **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of the perf event -++ * counter is selected when *map* is updated with perf event file -++ * descriptors. The *map* is an array whose size is the number of -++ * available CPUs, and each cell contains a value relative to one -++ * CPU. The value to retrieve is indicated by *flags*, that -++ * contains the index of the CPU to look up, masked with -++ * **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to -++ * **BPF_F_CURRENT_CPU** to indicate that the value for the -++ * current CPU should be retrieved. -++ * -++ * This helper behaves in a way close to -++ * **bpf_perf_event_read**\ () helper, save that instead of -++ * just returning the value observed, it fills the *buf* -++ * structure. This allows for additional data to be retrieved: in -++ * particular, the enabled and running times (in *buf*\ -++ * **->enabled** and *buf*\ **->running**, respectively) are -++ * copied. In general, **bpf_perf_event_read_value**\ () is -++ * recommended over **bpf_perf_event_read**\ (), which has some -++ * ABI issues and provides fewer functionalities. -++ * -++ * These values are interesting, because hardware PMU (Performance -++ * Monitoring Unit) counters are limited resources. When there are -++ * more PMU based perf events opened than available counters, -++ * kernel will multiplex these events so each event gets certain -++ * percentage (but not all) of the PMU time. In case that -++ * multiplexing happens, the number of samples or counter value -++ * will not reflect the case compared to when no multiplexing -++ * occurs. This makes comparison between different runs difficult. -++ * Typically, the counter value should be normalized before -++ * comparing to other experiments. The usual normalization is done -++ * as follows. -++ * -++ * :: -++ * -++ * normalized_counter = counter * t_enabled / t_running -++ * -++ * Where t_enabled is the time enabled for event and t_running is -++ * the time running for event since last normalization. The -++ * enabled and running times are accumulated since the perf event -++ * open. To achieve scaling factor between two invocations of an -++ * eBPF program, users can can use CPU id as the key (which is -++ * typical for perf array usage model) to remember the previous -++ * value and do the calculation inside the eBPF program. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_perf_prog_read_value(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, u32 buf_size) -++ * Description -++ * For en eBPF program attached to a perf event, retrieve the -++ * value of the event counter associated to *ctx* and store it in -++ * the structure pointed by *buf* and of size *buf_size*. Enabled -++ * and running times are also stored in the structure (see -++ * description of helper **bpf_perf_event_read_value**\ () for -++ * more details). -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_getsockopt(struct bpf_sock_ops *bpf_socket, int level, int optname, char *optval, int optlen) -++ * Description -++ * Emulate a call to **getsockopt()** on the socket associated to -++ * *bpf_socket*, which must be a full socket. The *level* at -++ * which the option resides and the name *optname* of the option -++ * must be specified, see **getsockopt(2)** for more information. -++ * The retrieved value is stored in the structure pointed by -++ * *opval* and of length *optlen*. -++ * -++ * This helper actually implements a subset of **getsockopt()**. -++ * It supports the following *level*\ s: -++ * -++ * * **IPPROTO_TCP**, which supports *optname* -++ * **TCP_CONGESTION**. -++ * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. -++ * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_override_return(struct pt_regs *regs, u64 rc) -++ * Description -++ * Used for error injection, this helper uses kprobes to override -++ * the return value of the probed function, and to set it to *rc*. -++ * The first argument is the context *regs* on which the kprobe -++ * works. -++ * -++ * This helper works by setting setting the PC (program counter) -++ * to an override function which is run in place of the original -++ * probed function. This means the probed function is not run at -++ * all. The replacement function just returns with the required -++ * value. -++ * -++ * This helper has security implications, and thus is subject to -++ * restrictions. It is only available if the kernel was compiled -++ * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration -++ * option, and in this case it only works on functions tagged with -++ * **ALLOW_ERROR_INJECTION** in the kernel code. -++ * -++ * Also, the helper is only available for the architectures having -++ * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing, -++ * x86 architecture is the only one to support this feature. -++ * Return -++ * 0 -++ * -++ * int bpf_sock_ops_cb_flags_set(struct bpf_sock_ops *bpf_sock, int argval) -++ * Description -++ * Attempt to set the value of the **bpf_sock_ops_cb_flags** field -++ * for the full TCP socket associated to *bpf_sock_ops* to -++ * *argval*. -++ * -++ * The primary use of this field is to determine if there should -++ * be calls to eBPF programs of type -++ * **BPF_PROG_TYPE_SOCK_OPS** at various points in the TCP -++ * code. A program of the same type can change its value, per -++ * connection and as necessary, when the connection is -++ * established. This field is directly accessible for reading, but -++ * this helper must be used for updates in order to return an -++ * error if an eBPF program tries to set a callback that is not -++ * supported in the current kernel. -++ * -++ * *argval* is a flag array which can combine these flags: -++ * -++ * * **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out) -++ * * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission) -++ * * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change) -++ * * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT) -++ * -++ * Therefore, this function can be used to clear a callback flag by -++ * setting the appropriate bit to zero. e.g. to disable the RTO -++ * callback: -++ * -++ * **bpf_sock_ops_cb_flags_set(bpf_sock,** -++ * **bpf_sock->bpf_sock_ops_cb_flags & ~BPF_SOCK_OPS_RTO_CB_FLAG)** -++ * -++ * Here are some examples of where one could call such eBPF -++ * program: -++ * -++ * * When RTO fires. -++ * * When a packet is retransmitted. -++ * * When the connection terminates. -++ * * When a packet is sent. -++ * * When a packet is received. -++ * Return -++ * Code **-EINVAL** if the socket is not a full TCP socket; -++ * otherwise, a positive number containing the bits that could not -++ * be set is returned (which comes down to 0 if all bits were set -++ * as required). -++ * -++ * int bpf_msg_redirect_map(struct sk_msg_buff *msg, struct bpf_map *map, u32 key, u64 flags) -++ * Description -++ * This helper is used in programs implementing policies at the -++ * socket level. If the message *msg* is allowed to pass (i.e. if -++ * the verdict eBPF program returns **SK_PASS**), redirect it to -++ * the socket referenced by *map* (of type -++ * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and -++ * egress interfaces can be used for redirection. The -++ * **BPF_F_INGRESS** value in *flags* is used to make the -++ * distinction (ingress path is selected if the flag is present, -++ * egress path otherwise). This is the only flag supported for now. -++ * Return -++ * **SK_PASS** on success, or **SK_DROP** on error. -++ * -++ * int bpf_msg_apply_bytes(struct sk_msg_buff *msg, u32 bytes) -++ * Description -++ * For socket policies, apply the verdict of the eBPF program to -++ * the next *bytes* (number of bytes) of message *msg*. -++ * -++ * For example, this helper can be used in the following cases: -++ * -++ * * A single **sendmsg**\ () or **sendfile**\ () system call -++ * contains multiple logical messages that the eBPF program is -++ * supposed to read and for which it should apply a verdict. -++ * * An eBPF program only cares to read the first *bytes* of a -++ * *msg*. If the message has a large payload, then setting up -++ * and calling the eBPF program repeatedly for all bytes, even -++ * though the verdict is already known, would create unnecessary -++ * overhead. -++ * -++ * When called from within an eBPF program, the helper sets a -++ * counter internal to the BPF infrastructure, that is used to -++ * apply the last verdict to the next *bytes*. If *bytes* is -++ * smaller than the current data being processed from a -++ * **sendmsg**\ () or **sendfile**\ () system call, the first -++ * *bytes* will be sent and the eBPF program will be re-run with -++ * the pointer for start of data pointing to byte number *bytes* -++ * **+ 1**. If *bytes* is larger than the current data being -++ * processed, then the eBPF verdict will be applied to multiple -++ * **sendmsg**\ () or **sendfile**\ () calls until *bytes* are -++ * consumed. -++ * -++ * Note that if a socket closes with the internal counter holding -++ * a non-zero value, this is not a problem because data is not -++ * being buffered for *bytes* and is sent as it is received. -++ * Return -++ * 0 -++ * -++ * int bpf_msg_cork_bytes(struct sk_msg_buff *msg, u32 bytes) -++ * Description -++ * For socket policies, prevent the execution of the verdict eBPF -++ * program for message *msg* until *bytes* (byte number) have been -++ * accumulated. -++ * -++ * This can be used when one needs a specific number of bytes -++ * before a verdict can be assigned, even if the data spans -++ * multiple **sendmsg**\ () or **sendfile**\ () calls. The extreme -++ * case would be a user calling **sendmsg**\ () repeatedly with -++ * 1-byte long message segments. Obviously, this is bad for -++ * performance, but it is still valid. If the eBPF program needs -++ * *bytes* bytes to validate a header, this helper can be used to -++ * prevent the eBPF program to be called again until *bytes* have -++ * been accumulated. -++ * Return -++ * 0 -++ * -++ * int bpf_msg_pull_data(struct sk_msg_buff *msg, u32 start, u32 end, u64 flags) -++ * Description -++ * For socket policies, pull in non-linear data from user space -++ * for *msg* and set pointers *msg*\ **->data** and *msg*\ -++ * **->data_end** to *start* and *end* bytes offsets into *msg*, -++ * respectively. -++ * -++ * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a -++ * *msg* it can only parse data that the (**data**, **data_end**) -++ * pointers have already consumed. For **sendmsg**\ () hooks this -++ * is likely the first scatterlist element. But for calls relying -++ * on the **sendpage** handler (e.g. **sendfile**\ ()) this will -++ * be the range (**0**, **0**) because the data is shared with -++ * user space and by default the objective is to avoid allowing -++ * user space to modify data while (or after) eBPF verdict is -++ * being decided. This helper can be used to pull in data and to -++ * set the start and end pointer to given values. Data will be -++ * copied if necessary (i.e. if data was not linear and if start -++ * and end pointers do not point to the same chunk). -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_bind(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) -++ * Description -++ * Bind the socket associated to *ctx* to the address pointed by -++ * *addr*, of length *addr_len*. This allows for making outgoing -++ * connection from the desired IP address, which can be useful for -++ * example when all processes inside a cgroup should use one -++ * single IP address on a host that has multiple IP configured. -++ * -++ * This helper works for IPv4 and IPv6, TCP and UDP sockets. The -++ * domain (*addr*\ **->sa_family**) must be **AF_INET** (or -++ * **AF_INET6**). Looking for a free port to bind to can be -++ * expensive, therefore binding to port is not permitted by the -++ * helper: *addr*\ **->sin_port** (or **sin6_port**, respectively) -++ * must be set to zero. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_xdp_adjust_tail(struct xdp_buff *xdp_md, int delta) -++ * Description -++ * Adjust (move) *xdp_md*\ **->data_end** by *delta* bytes. It is -++ * only possible to shrink the packet as of this writing, -++ * therefore *delta* must be a negative integer. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_skb_get_xfrm_state(struct sk_buff *skb, u32 index, struct bpf_xfrm_state *xfrm_state, u32 size, u64 flags) -++ * Description -++ * Retrieve the XFRM state (IP transform framework, see also -++ * **ip-xfrm(8)**) at *index* in XFRM "security path" for *skb*. -++ * -++ * The retrieved value is stored in the **struct bpf_xfrm_state** -++ * pointed by *xfrm_state* and of length *size*. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * This helper is available only if the kernel was compiled with -++ * **CONFIG_XFRM** configuration option. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_get_stack(struct pt_regs *regs, void *buf, u32 size, u64 flags) -++ * Description -++ * Return a user or a kernel stack in bpf program provided buffer. -++ * To achieve this, the helper needs *ctx*, which is a pointer -++ * to the context on which the tracing program is executed. -++ * To store the stacktrace, the bpf program provides *buf* with -++ * a nonnegative *size*. -++ * -++ * The last argument, *flags*, holds the number of stack frames to -++ * skip (from 0 to 255), masked with -++ * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set -++ * the following flags: -++ * -++ * **BPF_F_USER_STACK** -++ * Collect a user space stack instead of a kernel stack. -++ * **BPF_F_USER_BUILD_ID** -++ * Collect buildid+offset instead of ips for user stack, -++ * only valid if **BPF_F_USER_STACK** is also specified. -++ * -++ * **bpf_get_stack**\ () can collect up to -++ * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject -++ * to sufficient large buffer size. Note that -++ * this limit can be controlled with the **sysctl** program, and -++ * that it should be manually increased in order to profile long -++ * user stacks (such as stacks for Java programs). To do so, use: -++ * -++ * :: -++ * -++ * # sysctl kernel.perf_event_max_stack= -++ * Return -++ * A non-negative value equal to or less than *size* on success, -++ * or a negative error in case of failure. -++ * -++ * int bpf_skb_load_bytes_relative(const struct sk_buff *skb, u32 offset, void *to, u32 len, u32 start_header) -++ * Description -++ * This helper is similar to **bpf_skb_load_bytes**\ () in that -++ * it provides an easy way to load *len* bytes from *offset* -++ * from the packet associated to *skb*, into the buffer pointed -++ * by *to*. The difference to **bpf_skb_load_bytes**\ () is that -++ * a fifth argument *start_header* exists in order to select a -++ * base offset to start from. *start_header* can be one of: -++ * -++ * **BPF_HDR_START_MAC** -++ * Base offset to load data from is *skb*'s mac header. -++ * **BPF_HDR_START_NET** -++ * Base offset to load data from is *skb*'s network header. -++ * -++ * In general, "direct packet access" is the preferred method to -++ * access packet data, however, this helper is in particular useful -++ * in socket filters where *skb*\ **->data** does not always point -++ * to the start of the mac header and where "direct packet access" -++ * is not available. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_fib_lookup(void *ctx, struct bpf_fib_lookup *params, int plen, u32 flags) -++ * Description -++ * Do FIB lookup in kernel tables using parameters in *params*. -++ * If lookup is successful and result shows packet is to be -++ * forwarded, the neighbor tables are searched for the nexthop. -++ * If successful (ie., FIB lookup shows forwarding and nexthop -++ * is resolved), the nexthop address is returned in ipv4_dst -++ * or ipv6_dst based on family, smac is set to mac address of -++ * egress device, dmac is set to nexthop mac address, rt_metric -++ * is set to metric from route (IPv4/IPv6 only), and ifindex -++ * is set to the device index of the nexthop from the FIB lookup. -++ * -++ * *plen* argument is the size of the passed in struct. -++ * *flags* argument can be a combination of one or more of the -++ * following values: -++ * -++ * **BPF_FIB_LOOKUP_DIRECT** -++ * Do a direct table lookup vs full lookup using FIB -++ * rules. -++ * **BPF_FIB_LOOKUP_OUTPUT** -++ * Perform lookup from an egress perspective (default is -++ * ingress). -++ * -++ * *ctx* is either **struct xdp_md** for XDP programs or -++ * **struct sk_buff** tc cls_act programs. -++ * Return -++ * * < 0 if any input argument is invalid -++ * * 0 on success (packet is forwarded, nexthop neighbor exists) -++ * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the -++ * packet is not forwarded or needs assist from full stack -++ * -++ * int bpf_sock_hash_update(struct bpf_sock_ops_kern *skops, struct bpf_map *map, void *key, u64 flags) -++ * Description -++ * Add an entry to, or update a sockhash *map* referencing sockets. -++ * The *skops* is used as a new value for the entry associated to -++ * *key*. *flags* is one of: -++ * -++ * **BPF_NOEXIST** -++ * The entry for *key* must not exist in the map. -++ * **BPF_EXIST** -++ * The entry for *key* must already exist in the map. -++ * **BPF_ANY** -++ * No condition on the existence of the entry for *key*. -++ * -++ * If the *map* has eBPF programs (parser and verdict), those will -++ * be inherited by the socket being added. If the socket is -++ * already attached to eBPF programs, this results in an error. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_msg_redirect_hash(struct sk_msg_buff *msg, struct bpf_map *map, void *key, u64 flags) -++ * Description -++ * This helper is used in programs implementing policies at the -++ * socket level. If the message *msg* is allowed to pass (i.e. if -++ * the verdict eBPF program returns **SK_PASS**), redirect it to -++ * the socket referenced by *map* (of type -++ * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and -++ * egress interfaces can be used for redirection. The -++ * **BPF_F_INGRESS** value in *flags* is used to make the -++ * distinction (ingress path is selected if the flag is present, -++ * egress path otherwise). This is the only flag supported for now. -++ * Return -++ * **SK_PASS** on success, or **SK_DROP** on error. -++ * -++ * int bpf_sk_redirect_hash(struct sk_buff *skb, struct bpf_map *map, void *key, u64 flags) -++ * Description -++ * This helper is used in programs implementing policies at the -++ * skb socket level. If the sk_buff *skb* is allowed to pass (i.e. -++ * if the verdeict eBPF program returns **SK_PASS**), redirect it -++ * to the socket referenced by *map* (of type -++ * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and -++ * egress interfaces can be used for redirection. The -++ * **BPF_F_INGRESS** value in *flags* is used to make the -++ * distinction (ingress path is selected if the flag is present, -++ * egress otherwise). This is the only flag supported for now. -++ * Return -++ * **SK_PASS** on success, or **SK_DROP** on error. -++ * -++ * int bpf_lwt_push_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len) -++ * Description -++ * Encapsulate the packet associated to *skb* within a Layer 3 -++ * protocol header. This header is provided in the buffer at -++ * address *hdr*, with *len* its size in bytes. *type* indicates -++ * the protocol of the header and can be one of: -++ * -++ * **BPF_LWT_ENCAP_SEG6** -++ * IPv6 encapsulation with Segment Routing Header -++ * (**struct ipv6_sr_hdr**). *hdr* only contains the SRH, -++ * the IPv6 header is computed by the kernel. -++ * **BPF_LWT_ENCAP_SEG6_INLINE** -++ * Only works if *skb* contains an IPv6 packet. Insert a -++ * Segment Routing Header (**struct ipv6_sr_hdr**) inside -++ * the IPv6 header. -++ * **BPF_LWT_ENCAP_IP** -++ * IP encapsulation (GRE/GUE/IPIP/etc). The outer header -++ * must be IPv4 or IPv6, followed by zero or more -++ * additional headers, up to **LWT_BPF_MAX_HEADROOM** -++ * total bytes in all prepended headers. Please note that -++ * if **skb_is_gso**\ (*skb*) is true, no more than two -++ * headers can be prepended, and the inner header, if -++ * present, should be either GRE or UDP/GUE. -++ * -++ * **BPF_LWT_ENCAP_SEG6**\ \* types can be called by BPF programs -++ * of type **BPF_PROG_TYPE_LWT_IN**; **BPF_LWT_ENCAP_IP** type can -++ * be called by bpf programs of types **BPF_PROG_TYPE_LWT_IN** and -++ * **BPF_PROG_TYPE_LWT_XMIT**. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_lwt_seg6_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len) -++ * Description -++ * Store *len* bytes from address *from* into the packet -++ * associated to *skb*, at *offset*. Only the flags, tag and TLVs -++ * inside the outermost IPv6 Segment Routing Header can be -++ * modified through this helper. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_lwt_seg6_adjust_srh(struct sk_buff *skb, u32 offset, s32 delta) -++ * Description -++ * Adjust the size allocated to TLVs in the outermost IPv6 -++ * Segment Routing Header contained in the packet associated to -++ * *skb*, at position *offset* by *delta* bytes. Only offsets -++ * after the segments are accepted. *delta* can be as well -++ * positive (growing) as negative (shrinking). -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_lwt_seg6_action(struct sk_buff *skb, u32 action, void *param, u32 param_len) -++ * Description -++ * Apply an IPv6 Segment Routing action of type *action* to the -++ * packet associated to *skb*. Each action takes a parameter -++ * contained at address *param*, and of length *param_len* bytes. -++ * *action* can be one of: -++ * -++ * **SEG6_LOCAL_ACTION_END_X** -++ * End.X action: Endpoint with Layer-3 cross-connect. -++ * Type of *param*: **struct in6_addr**. -++ * **SEG6_LOCAL_ACTION_END_T** -++ * End.T action: Endpoint with specific IPv6 table lookup. -++ * Type of *param*: **int**. -++ * **SEG6_LOCAL_ACTION_END_B6** -++ * End.B6 action: Endpoint bound to an SRv6 policy. -++ * Type of *param*: **struct ipv6_sr_hdr**. -++ * **SEG6_LOCAL_ACTION_END_B6_ENCAP** -++ * End.B6.Encap action: Endpoint bound to an SRv6 -++ * encapsulation policy. -++ * Type of *param*: **struct ipv6_sr_hdr**. -++ * -++ * A call to this helper is susceptible to change the underlying -++ * packet buffer. Therefore, at load time, all checks on pointers -++ * previously done by the verifier are invalidated and must be -++ * performed again, if the helper is used in combination with -++ * direct packet access. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_rc_repeat(void *ctx) -++ * Description -++ * This helper is used in programs implementing IR decoding, to -++ * report a successfully decoded repeat key message. This delays -++ * the generation of a key up event for previously generated -++ * key down event. -++ * -++ * Some IR protocols like NEC have a special IR message for -++ * repeating last button, for when a button is held down. -++ * -++ * The *ctx* should point to the lirc sample as passed into -++ * the program. -++ * -++ * This helper is only available is the kernel was compiled with -++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to -++ * "**y**". -++ * Return -++ * 0 -++ * -++ * int bpf_rc_keydown(void *ctx, u32 protocol, u64 scancode, u32 toggle) -++ * Description -++ * This helper is used in programs implementing IR decoding, to -++ * report a successfully decoded key press with *scancode*, -++ * *toggle* value in the given *protocol*. The scancode will be -++ * translated to a keycode using the rc keymap, and reported as -++ * an input key down event. After a period a key up event is -++ * generated. This period can be extended by calling either -++ * **bpf_rc_keydown**\ () again with the same values, or calling -++ * **bpf_rc_repeat**\ (). -++ * -++ * Some protocols include a toggle bit, in case the button was -++ * released and pressed again between consecutive scancodes. -++ * -++ * The *ctx* should point to the lirc sample as passed into -++ * the program. -++ * -++ * The *protocol* is the decoded protocol number (see -++ * **enum rc_proto** for some predefined values). -++ * -++ * This helper is only available is the kernel was compiled with -++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to -++ * "**y**". -++ * Return -++ * 0 -++ * -++ * u64 bpf_skb_cgroup_id(struct sk_buff *skb) -++ * Description -++ * Return the cgroup v2 id of the socket associated with the *skb*. -++ * This is roughly similar to the **bpf_get_cgroup_classid**\ () -++ * helper for cgroup v1 by providing a tag resp. identifier that -++ * can be matched on or used for map lookups e.g. to implement -++ * policy. The cgroup v2 id of a given path in the hierarchy is -++ * exposed in user space through the f_handle API in order to get -++ * to the same 64-bit id. -++ * -++ * This helper can be used on TC egress path, but not on ingress, -++ * and is available only if the kernel was compiled with the -++ * **CONFIG_SOCK_CGROUP_DATA** configuration option. -++ * Return -++ * The id is returned or 0 in case the id could not be retrieved. -++ * -++ * u64 bpf_get_current_cgroup_id(void) -++ * Return -++ * A 64-bit integer containing the current cgroup id based -++ * on the cgroup within which the current task is running. -++ * -++ * void *bpf_get_local_storage(void *map, u64 flags) -++ * Description -++ * Get the pointer to the local storage area. -++ * The type and the size of the local storage is defined -++ * by the *map* argument. -++ * The *flags* meaning is specific for each map type, -++ * and has to be 0 for cgroup local storage. -++ * -++ * Depending on the BPF program type, a local storage area -++ * can be shared between multiple instances of the BPF program, -++ * running simultaneously. -++ * -++ * A user should care about the synchronization by himself. -++ * For example, by using the **BPF_STX_XADD** instruction to alter -++ * the shared data. -++ * Return -++ * A pointer to the local storage area. -++ * -++ * int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) -++ * Description -++ * Select a **SO_REUSEPORT** socket from a -++ * **BPF_MAP_TYPE_REUSEPORT_ARRAY** *map*. -++ * It checks the selected socket is matching the incoming -++ * request in the socket buffer. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * u64 bpf_skb_ancestor_cgroup_id(struct sk_buff *skb, int ancestor_level) -++ * Description -++ * Return id of cgroup v2 that is ancestor of cgroup associated -++ * with the *skb* at the *ancestor_level*. The root cgroup is at -++ * *ancestor_level* zero and each step down the hierarchy -++ * increments the level. If *ancestor_level* == level of cgroup -++ * associated with *skb*, then return value will be same as that -++ * of **bpf_skb_cgroup_id**\ (). -++ * -++ * The helper is useful to implement policies based on cgroups -++ * that are upper in hierarchy than immediate cgroup associated -++ * with *skb*. -++ * -++ * The format of returned id and helper limitations are same as in -++ * **bpf_skb_cgroup_id**\ (). -++ * Return -++ * The id is returned or 0 in case the id could not be retrieved. -++ * -++ * struct bpf_sock *bpf_sk_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) -++ * Description -++ * Look for TCP socket matching *tuple*, optionally in a child -++ * network namespace *netns*. The return value must be checked, -++ * and if non-**NULL**, released via **bpf_sk_release**\ (). -++ * -++ * The *ctx* should point to the context of the program, such as -++ * the skb or socket (depending on the hook in use). This is used -++ * to determine the base network namespace for the lookup. -++ * -++ * *tuple_size* must be one of: -++ * -++ * **sizeof**\ (*tuple*\ **->ipv4**) -++ * Look for an IPv4 socket. -++ * **sizeof**\ (*tuple*\ **->ipv6**) -++ * Look for an IPv6 socket. -++ * -++ * If the *netns* is a negative signed 32-bit integer, then the -++ * socket lookup table in the netns associated with the *ctx* will -++ * will be used. For the TC hooks, this is the netns of the device -++ * in the skb. For socket hooks, this is the netns of the socket. -++ * If *netns* is any other signed 32-bit value greater than or -++ * equal to zero then it specifies the ID of the netns relative to -++ * the netns associated with the *ctx*. *netns* values beyond the -++ * range of 32-bit integers are reserved for future use. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * This helper is available only if the kernel was compiled with -++ * **CONFIG_NET** configuration option. -++ * Return -++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. -++ * For sockets with reuseport option, the **struct bpf_sock** -++ * result is from *reuse*\ **->socks**\ [] using the hash of the -++ * tuple. -++ * -++ * struct bpf_sock *bpf_sk_lookup_udp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) -++ * Description -++ * Look for UDP socket matching *tuple*, optionally in a child -++ * network namespace *netns*. The return value must be checked, -++ * and if non-**NULL**, released via **bpf_sk_release**\ (). -++ * -++ * The *ctx* should point to the context of the program, such as -++ * the skb or socket (depending on the hook in use). This is used -++ * to determine the base network namespace for the lookup. -++ * -++ * *tuple_size* must be one of: -++ * -++ * **sizeof**\ (*tuple*\ **->ipv4**) -++ * Look for an IPv4 socket. -++ * **sizeof**\ (*tuple*\ **->ipv6**) -++ * Look for an IPv6 socket. -++ * -++ * If the *netns* is a negative signed 32-bit integer, then the -++ * socket lookup table in the netns associated with the *ctx* will -++ * will be used. For the TC hooks, this is the netns of the device -++ * in the skb. For socket hooks, this is the netns of the socket. -++ * If *netns* is any other signed 32-bit value greater than or -++ * equal to zero then it specifies the ID of the netns relative to -++ * the netns associated with the *ctx*. *netns* values beyond the -++ * range of 32-bit integers are reserved for future use. -++ * -++ * All values for *flags* are reserved for future usage, and must -++ * be left at zero. -++ * -++ * This helper is available only if the kernel was compiled with -++ * **CONFIG_NET** configuration option. -++ * Return -++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. -++ * For sockets with reuseport option, the **struct bpf_sock** -++ * result is from *reuse*\ **->socks**\ [] using the hash of the -++ * tuple. -++ * -++ * int bpf_sk_release(struct bpf_sock *sock) -++ * Description -++ * Release the reference held by *sock*. *sock* must be a -++ * non-**NULL** pointer that was returned from -++ * **bpf_sk_lookup_xxx**\ (). -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_map_push_elem(struct bpf_map *map, const void *value, u64 flags) -++ * Description -++ * Push an element *value* in *map*. *flags* is one of: -++ * -++ * **BPF_EXIST** -++ * If the queue/stack is full, the oldest element is -++ * removed to make room for this. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_map_pop_elem(struct bpf_map *map, void *value) -++ * Description -++ * Pop an element from *map*. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_map_peek_elem(struct bpf_map *map, void *value) -++ * Description -++ * Get an element from *map* without removing it. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_msg_push_data(struct sk_buff *skb, u32 start, u32 len, u64 flags) -++ * Description -++ * For socket policies, insert *len* bytes into *msg* at offset -++ * *start*. -++ * -++ * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a -++ * *msg* it may want to insert metadata or options into the *msg*. -++ * This can later be read and used by any of the lower layer BPF -++ * hooks. -++ * -++ * This helper may fail if under memory pressure (a malloc -++ * fails) in these cases BPF programs will get an appropriate -++ * error and BPF programs will need to handle them. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags) -++ * Description -++ * Will remove *pop* bytes from a *msg* starting at byte *start*. -++ * This may result in **ENOMEM** errors under certain situations if -++ * an allocation and copy are required due to a full ring buffer. -++ * However, the helper will try to avoid doing the allocation -++ * if possible. Other errors can occur if input parameters are -++ * invalid either due to *start* byte not being valid part of *msg* -++ * payload and/or *pop* value being to large. -++ * Return -++ * 0 on success, or a negative error in case of failure. -++ * -++ * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y) -++ * Description -++ * This helper is used in programs implementing IR decoding, to -++ * report a successfully decoded pointer movement. -++ * -++ * The *ctx* should point to the lirc sample as passed into -++ * the program. -++ * -++ * This helper is only available is the kernel was compiled with -++ * the **CONFIG_BPF_LIRC_MODE2** configuration option set to -++ * "**y**". -++ * Return -++ * 0 -++ * -++ * int bpf_spin_lock(struct bpf_spin_lock *lock) -++ * Description -++ * Acquire a spinlock represented by the pointer *lock*, which is -++ * stored as part of a value of a map. Taking the lock allows to -++ * safely update the rest of the fields in that value. The -++ * spinlock can (and must) later be released with a call to -++ * **bpf_spin_unlock**\ (\ *lock*\ ). -++ * -++ * Spinlocks in BPF programs come with a number of restrictions -++ * and constraints: -++ * -++ * * **bpf_spin_lock** objects are only allowed inside maps of -++ * types **BPF_MAP_TYPE_HASH** and **BPF_MAP_TYPE_ARRAY** (this -++ * list could be extended in the future). -++ * * BTF description of the map is mandatory. -++ * * The BPF program can take ONE lock at a time, since taking two -++ * or more could cause dead locks. -++ * * Only one **struct bpf_spin_lock** is allowed per map element. -++ * * When the lock is taken, calls (either BPF to BPF or helpers) -++ * are not allowed. -++ * * The **BPF_LD_ABS** and **BPF_LD_IND** instructions are not -++ * allowed inside a spinlock-ed region. -++ * * The BPF program MUST call **bpf_spin_unlock**\ () to release -++ * the lock, on all execution paths, before it returns. -++ * * The BPF program can access **struct bpf_spin_lock** only via -++ * the **bpf_spin_lock**\ () and **bpf_spin_unlock**\ () -++ * helpers. Loading or storing data into the **struct -++ * bpf_spin_lock** *lock*\ **;** field of a map is not allowed. -++ * * To use the **bpf_spin_lock**\ () helper, the BTF description -++ * of the map value must be a struct and have **struct -++ * bpf_spin_lock** *anyname*\ **;** field at the top level. -++ * Nested lock inside another struct is not allowed. -++ * * The **struct bpf_spin_lock** *lock* field in a map value must -++ * be aligned on a multiple of 4 bytes in that value. -++ * * Syscall with command **BPF_MAP_LOOKUP_ELEM** does not copy -++ * the **bpf_spin_lock** field to user space. -++ * * Syscall with command **BPF_MAP_UPDATE_ELEM**, or update from -++ * a BPF program, do not update the **bpf_spin_lock** field. -++ * * **bpf_spin_lock** cannot be on the stack or inside a -++ * networking packet (it can only be inside of a map values). -++ * * **bpf_spin_lock** is available to root only. -++ * * Tracing programs and socket filter programs cannot use -++ * **bpf_spin_lock**\ () due to insufficient preemption checks -++ * (but this may change in the future). -++ * * **bpf_spin_lock** is not allowed in inner maps of map-in-map. -++ * Return -++ * 0 -++ * -++ * int bpf_spin_unlock(struct bpf_spin_lock *lock) -++ * Description -++ * Release the *lock* previously locked by a call to -++ * **bpf_spin_lock**\ (\ *lock*\ ). -++ * Return -++ * 0 -++ * -++ * struct bpf_sock *bpf_sk_fullsock(struct bpf_sock *sk) -++ * Description -++ * This helper gets a **struct bpf_sock** pointer such -++ * that all the fields in this **bpf_sock** can be accessed. -++ * Return -++ * A **struct bpf_sock** pointer on success, or **NULL** in -++ * case of failure. -++ * -++ * struct bpf_tcp_sock *bpf_tcp_sock(struct bpf_sock *sk) -++ * Description -++ * This helper gets a **struct bpf_tcp_sock** pointer from a -++ * **struct bpf_sock** pointer. -++ * Return -++ * A **struct bpf_tcp_sock** pointer on success, or **NULL** in -++ * case of failure. -++ * -++ * int bpf_skb_ecn_set_ce(struct sk_buf *skb) -++ * Description -++ * Set ECN (Explicit Congestion Notification) field of IP header -++ * to **CE** (Congestion Encountered) if current value is **ECT** -++ * (ECN Capable Transport). Otherwise, do nothing. Works with IPv6 -++ * and IPv4. -++ * Return -++ * 1 if the **CE** flag is set (either by the current helper call -++ * or because it was already present), 0 if it is not set. -++ * -++ * struct bpf_sock *bpf_get_listener_sock(struct bpf_sock *sk) -++ * Description -++ * Return a **struct bpf_sock** pointer in **TCP_LISTEN** state. -++ * **bpf_sk_release**\ () is unnecessary and not allowed. -++ * Return -++ * A **struct bpf_sock** pointer on success, or **NULL** in -++ * case of failure. -++ * -++ * struct bpf_sock *bpf_skc_lookup_tcp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u64 netns, u64 flags) -++ * Description -++ * Look for TCP socket matching *tuple*, optionally in a child -++ * network namespace *netns*. The return value must be checked, -++ * and if non-**NULL**, released via **bpf_sk_release**\ (). -++ * -++ * This function is identical to **bpf_sk_lookup_tcp**\ (), except -++ * that it also returns timewait or request sockets. Use -++ * **bpf_sk_fullsock**\ () or **bpf_tcp_sock**\ () to access the -++ * full structure. -++ * -++ * This helper is available only if the kernel was compiled with -++ * **CONFIG_NET** configuration option. -++ * Return -++ * Pointer to **struct bpf_sock**, or **NULL** in case of failure. -++ * For sockets with reuseport option, the **struct bpf_sock** -++ * result is from *reuse*\ **->socks**\ [] using the hash of the -++ * tuple. -++ * -++ * int bpf_tcp_check_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) -++ * Description -++ * Check whether *iph* and *th* contain a valid SYN cookie ACK for -++ * the listening socket in *sk*. -++ * -++ * *iph* points to the start of the IPv4 or IPv6 header, while -++ * *iph_len* contains **sizeof**\ (**struct iphdr**) or -++ * **sizeof**\ (**struct ip6hdr**). -++ * -++ * *th* points to the start of the TCP header, while *th_len* -++ * contains **sizeof**\ (**struct tcphdr**). -++ * -++ * Return -++ * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative -++ * error otherwise. -++ * -++ * int bpf_sysctl_get_name(struct bpf_sysctl *ctx, char *buf, size_t buf_len, u64 flags) -++ * Description -++ * Get name of sysctl in /proc/sys/ and copy it into provided by -++ * program buffer *buf* of size *buf_len*. -++ * -++ * The buffer is always NUL terminated, unless it's zero-sized. -++ * -++ * If *flags* is zero, full name (e.g. "net/ipv4/tcp_mem") is -++ * copied. Use **BPF_F_SYSCTL_BASE_NAME** flag to copy base name -++ * only (e.g. "tcp_mem"). -++ * Return -++ * Number of character copied (not including the trailing NUL). -++ * -++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain -++ * truncated name in this case). -++ * -++ * int bpf_sysctl_get_current_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) -++ * Description -++ * Get current value of sysctl as it is presented in /proc/sys -++ * (incl. newline, etc), and copy it as a string into provided -++ * by program buffer *buf* of size *buf_len*. -++ * -++ * The whole value is copied, no matter what file position user -++ * space issued e.g. sys_read at. -++ * -++ * The buffer is always NUL terminated, unless it's zero-sized. -++ * Return -++ * Number of character copied (not including the trailing NUL). -++ * -++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain -++ * truncated name in this case). -++ * -++ * **-EINVAL** if current value was unavailable, e.g. because -++ * sysctl is uninitialized and read returns -EIO for it. -++ * -++ * int bpf_sysctl_get_new_value(struct bpf_sysctl *ctx, char *buf, size_t buf_len) -++ * Description -++ * Get new value being written by user space to sysctl (before -++ * the actual write happens) and copy it as a string into -++ * provided by program buffer *buf* of size *buf_len*. -++ * -++ * User space may write new value at file position > 0. -++ * -++ * The buffer is always NUL terminated, unless it's zero-sized. -++ * Return -++ * Number of character copied (not including the trailing NUL). -++ * -++ * **-E2BIG** if the buffer wasn't big enough (*buf* will contain -++ * truncated name in this case). -++ * -++ * **-EINVAL** if sysctl is being read. -++ * -++ * int bpf_sysctl_set_new_value(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) -++ * Description -++ * Override new value being written by user space to sysctl with -++ * value provided by program in buffer *buf* of size *buf_len*. -++ * -++ * *buf* should contain a string in same form as provided by user -++ * space on sysctl write. -++ * -++ * User space may write new value at file position > 0. To override -++ * the whole sysctl value file position should be set to zero. -++ * Return -++ * 0 on success. -++ * -++ * **-E2BIG** if the *buf_len* is too big. -++ * -++ * **-EINVAL** if sysctl is being read. -++ * -++ * int bpf_strtol(const char *buf, size_t buf_len, u64 flags, long *res) -++ * Description -++ * Convert the initial part of the string from buffer *buf* of -++ * size *buf_len* to a long integer according to the given base -++ * and save the result in *res*. -++ * -++ * The string may begin with an arbitrary amount of white space -++ * (as determined by **isspace**\ (3)) followed by a single -++ * optional '**-**' sign. -++ * -++ * Five least significant bits of *flags* encode base, other bits -++ * are currently unused. -++ * -++ * Base must be either 8, 10, 16 or 0 to detect it automatically -++ * similar to user space **strtol**\ (3). -++ * Return -++ * Number of characters consumed on success. Must be positive but -++ * no more than *buf_len*. -++ * -++ * **-EINVAL** if no valid digits were found or unsupported base -++ * was provided. -++ * -++ * **-ERANGE** if resulting value was out of range. -++ * -++ * int bpf_strtoul(const char *buf, size_t buf_len, u64 flags, unsigned long *res) -++ * Description -++ * Convert the initial part of the string from buffer *buf* of -++ * size *buf_len* to an unsigned long integer according to the -++ * given base and save the result in *res*. -++ * -++ * The string may begin with an arbitrary amount of white space -++ * (as determined by **isspace**\ (3)). -++ * -++ * Five least significant bits of *flags* encode base, other bits -++ * are currently unused. -++ * -++ * Base must be either 8, 10, 16 or 0 to detect it automatically -++ * similar to user space **strtoul**\ (3). -++ * Return -++ * Number of characters consumed on success. Must be positive but -++ * no more than *buf_len*. -++ * -++ * **-EINVAL** if no valid digits were found or unsupported base -++ * was provided. -++ * -++ * **-ERANGE** if resulting value was out of range. -++ * -++ * void *bpf_sk_storage_get(struct bpf_map *map, struct bpf_sock *sk, void *value, u64 flags) -++ * Description -++ * Get a bpf-local-storage from a *sk*. -++ * -++ * Logically, it could be thought of getting the value from -++ * a *map* with *sk* as the **key**. From this -++ * perspective, the usage is not much different from -++ * **bpf_map_lookup_elem**\ (*map*, **&**\ *sk*) except this -++ * helper enforces the key must be a full socket and the map must -++ * be a **BPF_MAP_TYPE_SK_STORAGE** also. -++ * -++ * Underneath, the value is stored locally at *sk* instead of -++ * the *map*. The *map* is used as the bpf-local-storage -++ * "type". The bpf-local-storage "type" (i.e. the *map*) is -++ * searched against all bpf-local-storages residing at *sk*. -++ * -++ * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be -++ * used such that a new bpf-local-storage will be -++ * created if one does not exist. *value* can be used -++ * together with **BPF_SK_STORAGE_GET_F_CREATE** to specify -++ * the initial value of a bpf-local-storage. If *value* is -++ * **NULL**, the new bpf-local-storage will be zero initialized. -++ * Return -++ * A bpf-local-storage pointer is returned on success. -++ * -++ * **NULL** if not found or there was an error in adding -++ * a new bpf-local-storage. -++ * -++ * int bpf_sk_storage_delete(struct bpf_map *map, struct bpf_sock *sk) -++ * Description -++ * Delete a bpf-local-storage from a *sk*. -++ * Return -++ * 0 on success. -++ * -++ * **-ENOENT** if the bpf-local-storage cannot be found. -++ * -++ * int bpf_send_signal(u32 sig) -++ * Description -++ * Send signal *sig* to the current task. -++ * Return -++ * 0 on success or successfully queued. -++ * -++ * **-EBUSY** if work queue under nmi is full. -++ * -++ * **-EINVAL** if *sig* is invalid. -++ * -++ * **-EPERM** if no permission to send the *sig*. -++ * -++ * **-EAGAIN** if bpf program can try again. -++ * -++ * s64 bpf_tcp_gen_syncookie(struct bpf_sock *sk, void *iph, u32 iph_len, struct tcphdr *th, u32 th_len) -++ * Description -++ * Try to issue a SYN cookie for the packet with corresponding -++ * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. -++ * -++ * *iph* points to the start of the IPv4 or IPv6 header, while -++ * *iph_len* contains **sizeof**\ (**struct iphdr**) or -++ * **sizeof**\ (**struct ip6hdr**). -++ * -++ * *th* points to the start of the TCP header, while *th_len* -++ * contains the length of the TCP header. -++ * -++ * Return -++ * On success, lower 32 bits hold the generated SYN cookie in -++ * followed by 16 bits which hold the MSS value for that cookie, -++ * and the top 16 bits are unused. -++ * -++ * On failure, the returned value is one of the following: -++ * -++ * **-EINVAL** SYN cookie cannot be issued due to error -++ * -++ * **-ENOENT** SYN cookie should not be issued (no SYN flood) -++ * -++ * **-EOPNOTSUPP** kernel configuration does not enable SYN cookies -++ * -++ * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 -++ */ -++#define __BPF_FUNC_MAPPER(FN) \ -++ FN(unspec), \ -++ FN(map_lookup_elem), \ -++ FN(map_update_elem), \ -++ FN(map_delete_elem), \ -++ FN(probe_read), \ -++ FN(ktime_get_ns), \ -++ FN(trace_printk), \ -++ FN(get_prandom_u32), \ -++ FN(get_smp_processor_id), \ -++ FN(skb_store_bytes), \ -++ FN(l3_csum_replace), \ -++ FN(l4_csum_replace), \ -++ FN(tail_call), \ -++ FN(clone_redirect), \ -++ FN(get_current_pid_tgid), \ -++ FN(get_current_uid_gid), \ -++ FN(get_current_comm), \ -++ FN(get_cgroup_classid), \ -++ FN(skb_vlan_push), \ -++ FN(skb_vlan_pop), \ -++ FN(skb_get_tunnel_key), \ -++ FN(skb_set_tunnel_key), \ -++ FN(perf_event_read), \ -++ FN(redirect), \ -++ FN(get_route_realm), \ -++ FN(perf_event_output), \ -++ FN(skb_load_bytes), \ -++ FN(get_stackid), \ -++ FN(csum_diff), \ -++ FN(skb_get_tunnel_opt), \ -++ FN(skb_set_tunnel_opt), \ -++ FN(skb_change_proto), \ -++ FN(skb_change_type), \ -++ FN(skb_under_cgroup), \ -++ FN(get_hash_recalc), \ -++ FN(get_current_task), \ -++ FN(probe_write_user), \ -++ FN(current_task_under_cgroup), \ -++ FN(skb_change_tail), \ -++ FN(skb_pull_data), \ -++ FN(csum_update), \ -++ FN(set_hash_invalid), \ -++ FN(get_numa_node_id), \ -++ FN(skb_change_head), \ -++ FN(xdp_adjust_head), \ -++ FN(probe_read_str), \ -++ FN(get_socket_cookie), \ -++ FN(get_socket_uid), \ -++ FN(set_hash), \ -++ FN(setsockopt), \ -++ FN(skb_adjust_room), \ -++ FN(redirect_map), \ -++ FN(sk_redirect_map), \ -++ FN(sock_map_update), \ -++ FN(xdp_adjust_meta), \ -++ FN(perf_event_read_value), \ -++ FN(perf_prog_read_value), \ -++ FN(getsockopt), \ -++ FN(override_return), \ -++ FN(sock_ops_cb_flags_set), \ -++ FN(msg_redirect_map), \ -++ FN(msg_apply_bytes), \ -++ FN(msg_cork_bytes), \ -++ FN(msg_pull_data), \ -++ FN(bind), \ -++ FN(xdp_adjust_tail), \ -++ FN(skb_get_xfrm_state), \ -++ FN(get_stack), \ -++ FN(skb_load_bytes_relative), \ -++ FN(fib_lookup), \ -++ FN(sock_hash_update), \ -++ FN(msg_redirect_hash), \ -++ FN(sk_redirect_hash), \ -++ FN(lwt_push_encap), \ -++ FN(lwt_seg6_store_bytes), \ -++ FN(lwt_seg6_adjust_srh), \ -++ FN(lwt_seg6_action), \ -++ FN(rc_repeat), \ -++ FN(rc_keydown), \ -++ FN(skb_cgroup_id), \ -++ FN(get_current_cgroup_id), \ -++ FN(get_local_storage), \ -++ FN(sk_select_reuseport), \ -++ FN(skb_ancestor_cgroup_id), \ -++ FN(sk_lookup_tcp), \ -++ FN(sk_lookup_udp), \ -++ FN(sk_release), \ -++ FN(map_push_elem), \ -++ FN(map_pop_elem), \ -++ FN(map_peek_elem), \ -++ FN(msg_push_data), \ -++ FN(msg_pop_data), \ -++ FN(rc_pointer_rel), \ -++ FN(spin_lock), \ -++ FN(spin_unlock), \ -++ FN(sk_fullsock), \ -++ FN(tcp_sock), \ -++ FN(skb_ecn_set_ce), \ -++ FN(get_listener_sock), \ -++ FN(skc_lookup_tcp), \ -++ FN(tcp_check_syncookie), \ -++ FN(sysctl_get_name), \ -++ FN(sysctl_get_current_value), \ -++ FN(sysctl_get_new_value), \ -++ FN(sysctl_set_new_value), \ -++ FN(strtol), \ -++ FN(strtoul), \ -++ FN(sk_storage_get), \ -++ FN(sk_storage_delete), \ -++ FN(send_signal), \ -++ FN(tcp_gen_syncookie), -++ -+ /* integer value in 'imm' field of BPF_CALL instruction selects which helper -+ * function eBPF program intends to call -+ */ -++#define __BPF_ENUM_FN(x) BPF_FUNC_ ## x -+ enum bpf_func_id { -+- BPF_FUNC_unspec, -+- BPF_FUNC_map_lookup_elem, /* void *map_lookup_elem(&map, &key) */ -+- BPF_FUNC_map_update_elem, /* int map_update_elem(&map, &key, &value, flags) */ -+- BPF_FUNC_map_delete_elem, /* int map_delete_elem(&map, &key) */ -+- BPF_FUNC_probe_read, /* int bpf_probe_read(void *dst, int size, void *src) */ -+- BPF_FUNC_ktime_get_ns, /* u64 bpf_ktime_get_ns(void) */ -+- BPF_FUNC_trace_printk, /* int bpf_trace_printk(const char *fmt, int fmt_size, ...) */ -+- BPF_FUNC_get_prandom_u32, /* u32 prandom_u32(void) */ -+- BPF_FUNC_get_smp_processor_id, /* u32 raw_smp_processor_id(void) */ -+- -+- /** -+- * skb_store_bytes(skb, offset, from, len, flags) - store bytes into packet -+- * @skb: pointer to skb -+- * @offset: offset within packet from skb->mac_header -+- * @from: pointer where to copy bytes from -+- * @len: number of bytes to store into packet -+- * @flags: bit 0 - if true, recompute skb->csum -+- * other bits - reserved -+- * Return: 0 on success -+- */ -+- BPF_FUNC_skb_store_bytes, -+- -+- /** -+- * l3_csum_replace(skb, offset, from, to, flags) - recompute IP checksum -+- * @skb: pointer to skb -+- * @offset: offset within packet where IP checksum is located -+- * @from: old value of header field -+- * @to: new value of header field -+- * @flags: bits 0-3 - size of header field -+- * other bits - reserved -+- * Return: 0 on success -+- */ -+- BPF_FUNC_l3_csum_replace, -+- -+- /** -+- * l4_csum_replace(skb, offset, from, to, flags) - recompute TCP/UDP checksum -+- * @skb: pointer to skb -+- * @offset: offset within packet where TCP/UDP checksum is located -+- * @from: old value of header field -+- * @to: new value of header field -+- * @flags: bits 0-3 - size of header field -+- * bit 4 - is pseudo header -+- * other bits - reserved -+- * Return: 0 on success -+- */ -+- BPF_FUNC_l4_csum_replace, -++ __BPF_FUNC_MAPPER(__BPF_ENUM_FN) -++ __BPF_FUNC_MAX_ID, -++}; -++#undef __BPF_ENUM_FN -+ -+- /** -+- * bpf_tail_call(ctx, prog_array_map, index) - jump into another BPF program -+- * @ctx: context pointer passed to next program -+- * @prog_array_map: pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY -+- * @index: index inside array that selects specific program to run -+- * Return: 0 on success -+- */ -+- BPF_FUNC_tail_call, -++/* All flags used by eBPF helper functions, placed here. */ -+ -+- /** -+- * bpf_clone_redirect(skb, ifindex, flags) - redirect to another netdev -+- * @skb: pointer to skb -+- * @ifindex: ifindex of the net device -+- * @flags: bit 0 - if set, redirect to ingress instead of egress -+- * other bits - reserved -+- * Return: 0 on success -+- */ -+- BPF_FUNC_clone_redirect, -++/* BPF_FUNC_skb_store_bytes flags. */ -++#define BPF_F_RECOMPUTE_CSUM (1ULL << 0) -++#define BPF_F_INVALIDATE_HASH (1ULL << 1) -+ -+- /** -+- * u64 bpf_get_current_pid_tgid(void) -+- * Return: current->tgid << 32 | current->pid -+- */ -+- BPF_FUNC_get_current_pid_tgid, -++/* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags. -++ * First 4 bits are for passing the header field size. -++ */ -++#define BPF_F_HDR_FIELD_MASK 0xfULL -+ -+- /** -+- * u64 bpf_get_current_uid_gid(void) -+- * Return: current_gid << 32 | current_uid -+- */ -+- BPF_FUNC_get_current_uid_gid, -++/* BPF_FUNC_l4_csum_replace flags. */ -++#define BPF_F_PSEUDO_HDR (1ULL << 4) -++#define BPF_F_MARK_MANGLED_0 (1ULL << 5) -++#define BPF_F_MARK_ENFORCE (1ULL << 6) -++ -++/* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ -++#define BPF_F_INGRESS (1ULL << 0) -++ -++/* BPF_FUNC_skb_set_tunnel_key and BPF_FUNC_skb_get_tunnel_key flags. */ -++#define BPF_F_TUNINFO_IPV6 (1ULL << 0) -++ -++/* flags for both BPF_FUNC_get_stackid and BPF_FUNC_get_stack. */ -++#define BPF_F_SKIP_FIELD_MASK 0xffULL -++#define BPF_F_USER_STACK (1ULL << 8) -++/* flags used by BPF_FUNC_get_stackid only. */ -++#define BPF_F_FAST_STACK_CMP (1ULL << 9) -++#define BPF_F_REUSE_STACKID (1ULL << 10) -++/* flags used by BPF_FUNC_get_stack only. */ -++#define BPF_F_USER_BUILD_ID (1ULL << 11) -++ -++/* BPF_FUNC_skb_set_tunnel_key flags. */ -++#define BPF_F_ZERO_CSUM_TX (1ULL << 1) -++#define BPF_F_DONT_FRAGMENT (1ULL << 2) -++#define BPF_F_SEQ_NUMBER (1ULL << 3) -+ -+- /** -+- * bpf_get_current_comm(char *buf, int size_of_buf) -+- * stores current->comm into buf -+- * Return: 0 on success -+- */ -+- BPF_FUNC_get_current_comm, -+- -+- /** -+- * bpf_get_cgroup_classid(skb) - retrieve a proc's classid -+- * @skb: pointer to skb -+- * Return: classid if != 0 -+- */ -+- BPF_FUNC_get_cgroup_classid, -+- BPF_FUNC_skb_vlan_push, /* bpf_skb_vlan_push(skb, vlan_proto, vlan_tci) */ -+- BPF_FUNC_skb_vlan_pop, /* bpf_skb_vlan_pop(skb) */ -+- -+- /** -+- * bpf_skb_[gs]et_tunnel_key(skb, key, size, flags) -+- * retrieve or populate tunnel metadata -+- * @skb: pointer to skb -+- * @key: pointer to 'struct bpf_tunnel_key' -+- * @size: size of 'struct bpf_tunnel_key' -+- * @flags: room for future extensions -+- * Retrun: 0 on success -+- */ -+- BPF_FUNC_skb_get_tunnel_key, -+- BPF_FUNC_skb_set_tunnel_key, -+- BPF_FUNC_perf_event_read, /* u64 bpf_perf_event_read(&map, index) */ -+- /** -+- * bpf_redirect(ifindex, flags) - redirect to another netdev -+- * @ifindex: ifindex of the net device -+- * @flags: bit 0 - if set, redirect to ingress instead of egress -+- * other bits - reserved -+- * Return: TC_ACT_REDIRECT -+- */ -+- BPF_FUNC_redirect, -++/* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and -++ * BPF_FUNC_perf_event_read_value flags. -++ */ -++#define BPF_F_INDEX_MASK 0xffffffffULL -++#define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK -++/* BPF_FUNC_perf_event_output for sk_buff input context. */ -++#define BPF_F_CTXLEN_MASK (0xfffffULL << 32) -++ -++/* Current network namespace */ -++#define BPF_F_CURRENT_NETNS (-1L) -++ -++/* BPF_FUNC_skb_adjust_room flags. */ -++#define BPF_F_ADJ_ROOM_FIXED_GSO (1ULL << 0) -++ -++#define BPF_ADJ_ROOM_ENCAP_L2_MASK 0xff -++#define BPF_ADJ_ROOM_ENCAP_L2_SHIFT 56 -++ -++#define BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 (1ULL << 1) -++#define BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 (1ULL << 2) -++#define BPF_F_ADJ_ROOM_ENCAP_L4_GRE (1ULL << 3) -++#define BPF_F_ADJ_ROOM_ENCAP_L4_UDP (1ULL << 4) -++#define BPF_F_ADJ_ROOM_ENCAP_L2(len) (((__u64)len & \ -++ BPF_ADJ_ROOM_ENCAP_L2_MASK) \ -++ << BPF_ADJ_ROOM_ENCAP_L2_SHIFT) -++ -++/* BPF_FUNC_sysctl_get_name flags. */ -++#define BPF_F_SYSCTL_BASE_NAME (1ULL << 0) -++ -++/* BPF_FUNC_sk_storage_get flags */ -++#define BPF_SK_STORAGE_GET_F_CREATE (1ULL << 0) -++ -++/* Mode for BPF_FUNC_skb_adjust_room helper. */ -++enum bpf_adj_room_mode { -++ BPF_ADJ_ROOM_NET, -++ BPF_ADJ_ROOM_MAC, -++}; -+ -+- /** -+- * bpf_get_route_realm(skb) - retrieve a dst's tclassid -+- * @skb: pointer to skb -+- * Return: realm if != 0 -+- */ -+- BPF_FUNC_get_route_realm, -++/* Mode for BPF_FUNC_skb_load_bytes_relative helper. */ -++enum bpf_hdr_start_off { -++ BPF_HDR_START_MAC, -++ BPF_HDR_START_NET, -++}; -+ -+- /** -+- * bpf_perf_event_output(ctx, map, index, data, size) - output perf raw sample -+- * @ctx: struct pt_regs* -+- * @map: pointer to perf_event_array map -+- * @index: index of event in the map -+- * @data: data on stack to be output as raw data -+- * @size: size of data -+- * Return: 0 on success -+- */ -+- BPF_FUNC_perf_event_output, -+- __BPF_FUNC_MAX_ID, -++/* Encapsulation type for BPF_FUNC_lwt_push_encap helper. */ -++enum bpf_lwt_encap_mode { -++ BPF_LWT_ENCAP_SEG6, -++ BPF_LWT_ENCAP_SEG6_INLINE, -++ BPF_LWT_ENCAP_IP, -+ }; -+ -++#define __bpf_md_ptr(type, name) \ -++union { \ -++ type name; \ -++ __u64 :64; \ -++} __attribute__((aligned(8))) -++ -+ /* user accessible mirror of in-kernel sk_buff. -+ * new fields can only be added to the end of this structure -+ */ -+@@ -291,11 +2985,632 @@ struct __sk_buff { -+ __u32 cb[5]; -+ __u32 hash; -+ __u32 tc_classid; -++ __u32 data; -++ __u32 data_end; -++ __u32 napi_id; -++ -++ /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ -++ __u32 family; -++ __u32 remote_ip4; /* Stored in network byte order */ -++ __u32 local_ip4; /* Stored in network byte order */ -++ __u32 remote_ip6[4]; /* Stored in network byte order */ -++ __u32 local_ip6[4]; /* Stored in network byte order */ -++ __u32 remote_port; /* Stored in network byte order */ -++ __u32 local_port; /* stored in host byte order */ -++ /* ... here. */ -++ -++ __u32 data_meta; -++ __bpf_md_ptr(struct bpf_flow_keys *, flow_keys); -++ __u64 tstamp; -++ __u32 wire_len; -++ __u32 gso_segs; -++ __bpf_md_ptr(struct bpf_sock *, sk); -+ }; -+ -+ struct bpf_tunnel_key { -+ __u32 tunnel_id; -+- __u32 remote_ipv4; -++ union { -++ __u32 remote_ipv4; -++ __u32 remote_ipv6[4]; -++ }; -++ __u8 tunnel_tos; -++ __u8 tunnel_ttl; -++ __u16 tunnel_ext; /* Padding, future use. */ -++ __u32 tunnel_label; -++}; -++ -++/* user accessible mirror of in-kernel xfrm_state. -++ * new fields can only be added to the end of this structure -++ */ -++struct bpf_xfrm_state { -++ __u32 reqid; -++ __u32 spi; /* Stored in network byte order */ -++ __u16 family; -++ __u16 ext; /* Padding, future use. */ -++ union { -++ __u32 remote_ipv4; /* Stored in network byte order */ -++ __u32 remote_ipv6[4]; /* Stored in network byte order */ -++ }; -++}; -++ -++/* Generic BPF return codes which all BPF program types may support. -++ * The values are binary compatible with their TC_ACT_* counter-part to -++ * provide backwards compatibility with existing SCHED_CLS and SCHED_ACT -++ * programs. -++ * -++ * XDP is handled seprately, see XDP_*. -++ */ -++enum bpf_ret_code { -++ BPF_OK = 0, -++ /* 1 reserved */ -++ BPF_DROP = 2, -++ /* 3-6 reserved */ -++ BPF_REDIRECT = 7, -++ /* >127 are reserved for prog type specific return codes. -++ * -++ * BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and -++ * BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been -++ * changed and should be routed based on its new L3 header. -++ * (This is an L3 redirect, as opposed to L2 redirect -++ * represented by BPF_REDIRECT above). -++ */ -++ BPF_LWT_REROUTE = 128, -++}; -++ -++struct bpf_sock { -++ __u32 bound_dev_if; -++ __u32 family; -++ __u32 type; -++ __u32 protocol; -++ __u32 mark; -++ __u32 priority; -++ /* IP address also allows 1 and 2 bytes access */ -++ __u32 src_ip4; -++ __u32 src_ip6[4]; -++ __u32 src_port; /* host byte order */ -++ __u32 dst_port; /* network byte order */ -++ __u32 dst_ip4; -++ __u32 dst_ip6[4]; -++ __u32 state; -++}; -++ -++struct bpf_tcp_sock { -++ __u32 snd_cwnd; /* Sending congestion window */ -++ __u32 srtt_us; /* smoothed round trip time << 3 in usecs */ -++ __u32 rtt_min; -++ __u32 snd_ssthresh; /* Slow start size threshold */ -++ __u32 rcv_nxt; /* What we want to receive next */ -++ __u32 snd_nxt; /* Next sequence we send */ -++ __u32 snd_una; /* First byte we want an ack for */ -++ __u32 mss_cache; /* Cached effective mss, not including SACKS */ -++ __u32 ecn_flags; /* ECN status bits. */ -++ __u32 rate_delivered; /* saved rate sample: packets delivered */ -++ __u32 rate_interval_us; /* saved rate sample: time elapsed */ -++ __u32 packets_out; /* Packets which are "in flight" */ -++ __u32 retrans_out; /* Retransmitted packets out */ -++ __u32 total_retrans; /* Total retransmits for entire connection */ -++ __u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn -++ * total number of segments in. -++ */ -++ __u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn -++ * total number of data segments in. -++ */ -++ __u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut -++ * The total number of segments sent. -++ */ -++ __u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut -++ * total number of data segments sent. -++ */ -++ __u32 lost_out; /* Lost packets */ -++ __u32 sacked_out; /* SACK'd packets */ -++ __u64 bytes_received; /* RFC4898 tcpEStatsAppHCThruOctetsReceived -++ * sum(delta(rcv_nxt)), or how many bytes -++ * were acked. -++ */ -++ __u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked -++ * sum(delta(snd_una)), or how many bytes -++ * were acked. -++ */ -++ __u32 dsack_dups; /* RFC4898 tcpEStatsStackDSACKDups -++ * total number of DSACK blocks received -++ */ -++ __u32 delivered; /* Total data packets delivered incl. rexmits */ -++ __u32 delivered_ce; /* Like the above but only ECE marked packets */ -++ __u32 icsk_retransmits; /* Number of unrecovered [RTO] timeouts */ -++}; -++ -++struct bpf_sock_tuple { -++ union { -++ struct { -++ __be32 saddr; -++ __be32 daddr; -++ __be16 sport; -++ __be16 dport; -++ } ipv4; -++ struct { -++ __be32 saddr[4]; -++ __be32 daddr[4]; -++ __be16 sport; -++ __be16 dport; -++ } ipv6; -++ }; -++}; -++ -++struct bpf_xdp_sock { -++ __u32 queue_id; -++}; -++ -++#define XDP_PACKET_HEADROOM 256 -++ -++/* User return codes for XDP prog type. -++ * A valid XDP program must return one of these defined values. All other -++ * return codes are reserved for future use. Unknown return codes will -++ * result in packet drops and a warning via bpf_warn_invalid_xdp_action(). -++ */ -++enum xdp_action { -++ XDP_ABORTED = 0, -++ XDP_DROP, -++ XDP_PASS, -++ XDP_TX, -++ XDP_REDIRECT, -++}; -++ -++/* user accessible metadata for XDP packet hook -++ * new fields must be added to the end of this structure -++ */ -++struct xdp_md { -++ __u32 data; -++ __u32 data_end; -++ __u32 data_meta; -++ /* Below access go through struct xdp_rxq_info */ -++ __u32 ingress_ifindex; /* rxq->dev->ifindex */ -++ __u32 rx_queue_index; /* rxq->queue_index */ -++}; -++ -++enum sk_action { -++ SK_DROP = 0, -++ SK_PASS, -++}; -++ -++/* user accessible metadata for SK_MSG packet hook, new fields must -++ * be added to the end of this structure -++ */ -++struct sk_msg_md { -++ __bpf_md_ptr(void *, data); -++ __bpf_md_ptr(void *, data_end); -++ -++ __u32 family; -++ __u32 remote_ip4; /* Stored in network byte order */ -++ __u32 local_ip4; /* Stored in network byte order */ -++ __u32 remote_ip6[4]; /* Stored in network byte order */ -++ __u32 local_ip6[4]; /* Stored in network byte order */ -++ __u32 remote_port; /* Stored in network byte order */ -++ __u32 local_port; /* stored in host byte order */ -++ __u32 size; /* Total size of sk_msg */ -++}; -++ -++struct sk_reuseport_md { -++ /* -++ * Start of directly accessible data. It begins from -++ * the tcp/udp header. -++ */ -++ __bpf_md_ptr(void *, data); -++ /* End of directly accessible data */ -++ __bpf_md_ptr(void *, data_end); -++ /* -++ * Total length of packet (starting from the tcp/udp header). -++ * Note that the directly accessible bytes (data_end - data) -++ * could be less than this "len". Those bytes could be -++ * indirectly read by a helper "bpf_skb_load_bytes()". -++ */ -++ __u32 len; -++ /* -++ * Eth protocol in the mac header (network byte order). e.g. -++ * ETH_P_IP(0x0800) and ETH_P_IPV6(0x86DD) -++ */ -++ __u32 eth_protocol; -++ __u32 ip_protocol; /* IP protocol. e.g. IPPROTO_TCP, IPPROTO_UDP */ -++ __u32 bind_inany; /* Is sock bound to an INANY address? */ -++ __u32 hash; /* A hash of the packet 4 tuples */ -++}; -++ -++#define BPF_TAG_SIZE 8 -++ -++struct bpf_prog_info { -++ __u32 type; -++ __u32 id; -++ __u8 tag[BPF_TAG_SIZE]; -++ __u32 jited_prog_len; -++ __u32 xlated_prog_len; -++ __aligned_u64 jited_prog_insns; -++ __aligned_u64 xlated_prog_insns; -++ __u64 load_time; /* ns since boottime */ -++ __u32 created_by_uid; -++ __u32 nr_map_ids; -++ __aligned_u64 map_ids; -++ char name[BPF_OBJ_NAME_LEN]; -++ __u32 ifindex; -++ __u32 gpl_compatible:1; -++ __u32 :31; /* alignment pad */ -++ __u64 netns_dev; -++ __u64 netns_ino; -++ __u32 nr_jited_ksyms; -++ __u32 nr_jited_func_lens; -++ __aligned_u64 jited_ksyms; -++ __aligned_u64 jited_func_lens; -++ __u32 btf_id; -++ __u32 func_info_rec_size; -++ __aligned_u64 func_info; -++ __u32 nr_func_info; -++ __u32 nr_line_info; -++ __aligned_u64 line_info; -++ __aligned_u64 jited_line_info; -++ __u32 nr_jited_line_info; -++ __u32 line_info_rec_size; -++ __u32 jited_line_info_rec_size; -++ __u32 nr_prog_tags; -++ __aligned_u64 prog_tags; -++ __u64 run_time_ns; -++ __u64 run_cnt; -++} __attribute__((aligned(8))); -++ -++struct bpf_map_info { -++ __u32 type; -++ __u32 id; -++ __u32 key_size; -++ __u32 value_size; -++ __u32 max_entries; -++ __u32 map_flags; -++ char name[BPF_OBJ_NAME_LEN]; -++ __u32 ifindex; -++ __u32 :32; -++ __u64 netns_dev; -++ __u64 netns_ino; -++ __u32 btf_id; -++ __u32 btf_key_type_id; -++ __u32 btf_value_type_id; -++} __attribute__((aligned(8))); -++ -++struct bpf_btf_info { -++ __aligned_u64 btf; -++ __u32 btf_size; -++ __u32 id; -++} __attribute__((aligned(8))); -++ -++/* User bpf_sock_addr struct to access socket fields and sockaddr struct passed -++ * by user and intended to be used by socket (e.g. to bind to, depends on -++ * attach attach type). -++ */ -++struct bpf_sock_addr { -++ __u32 user_family; /* Allows 4-byte read, but no write. */ -++ __u32 user_ip4; /* Allows 1,2,4-byte read and 4-byte write. -++ * Stored in network byte order. -++ */ -++ __u32 user_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. -++ * Stored in network byte order. -++ */ -++ __u32 user_port; /* Allows 4-byte read and write. -++ * Stored in network byte order -++ */ -++ __u32 family; /* Allows 4-byte read, but no write */ -++ __u32 type; /* Allows 4-byte read, but no write */ -++ __u32 protocol; /* Allows 4-byte read, but no write */ -++ __u32 msg_src_ip4; /* Allows 1,2,4-byte read and 4-byte write. -++ * Stored in network byte order. -++ */ -++ __u32 msg_src_ip6[4]; /* Allows 1,2,4,8-byte read and 4,8-byte write. -++ * Stored in network byte order. -++ */ -++ __bpf_md_ptr(struct bpf_sock *, sk); -++}; -++ -++/* User bpf_sock_ops struct to access socket values and specify request ops -++ * and their replies. -++ * Some of this fields are in network (bigendian) byte order and may need -++ * to be converted before use (bpf_ntohl() defined in samples/bpf/bpf_endian.h). -++ * New fields can only be added at the end of this structure -++ */ -++struct bpf_sock_ops { -++ __u32 op; -++ union { -++ __u32 args[4]; /* Optionally passed to bpf program */ -++ __u32 reply; /* Returned by bpf program */ -++ __u32 replylong[4]; /* Optionally returned by bpf prog */ -++ }; -++ __u32 family; -++ __u32 remote_ip4; /* Stored in network byte order */ -++ __u32 local_ip4; /* Stored in network byte order */ -++ __u32 remote_ip6[4]; /* Stored in network byte order */ -++ __u32 local_ip6[4]; /* Stored in network byte order */ -++ __u32 remote_port; /* Stored in network byte order */ -++ __u32 local_port; /* stored in host byte order */ -++ __u32 is_fullsock; /* Some TCP fields are only valid if -++ * there is a full socket. If not, the -++ * fields read as zero. -++ */ -++ __u32 snd_cwnd; -++ __u32 srtt_us; /* Averaged RTT << 3 in usecs */ -++ __u32 bpf_sock_ops_cb_flags; /* flags defined in uapi/linux/tcp.h */ -++ __u32 state; -++ __u32 rtt_min; -++ __u32 snd_ssthresh; -++ __u32 rcv_nxt; -++ __u32 snd_nxt; -++ __u32 snd_una; -++ __u32 mss_cache; -++ __u32 ecn_flags; -++ __u32 rate_delivered; -++ __u32 rate_interval_us; -++ __u32 packets_out; -++ __u32 retrans_out; -++ __u32 total_retrans; -++ __u32 segs_in; -++ __u32 data_segs_in; -++ __u32 segs_out; -++ __u32 data_segs_out; -++ __u32 lost_out; -++ __u32 sacked_out; -++ __u32 sk_txhash; -++ __u64 bytes_received; -++ __u64 bytes_acked; -++ __bpf_md_ptr(struct bpf_sock *, sk); -++}; -++ -++/* Definitions for bpf_sock_ops_cb_flags */ -++#define BPF_SOCK_OPS_RTO_CB_FLAG (1<<0) -++#define BPF_SOCK_OPS_RETRANS_CB_FLAG (1<<1) -++#define BPF_SOCK_OPS_STATE_CB_FLAG (1<<2) -++#define BPF_SOCK_OPS_RTT_CB_FLAG (1<<3) -++#define BPF_SOCK_OPS_ALL_CB_FLAGS 0xF /* Mask of all currently -++ * supported cb flags -++ */ -++ -++/* List of known BPF sock_ops operators. -++ * New entries can only be added at the end -++ */ -++enum { -++ BPF_SOCK_OPS_VOID, -++ BPF_SOCK_OPS_TIMEOUT_INIT, /* Should return SYN-RTO value to use or -++ * -1 if default value should be used -++ */ -++ BPF_SOCK_OPS_RWND_INIT, /* Should return initial advertized -++ * window (in packets) or -1 if default -++ * value should be used -++ */ -++ BPF_SOCK_OPS_TCP_CONNECT_CB, /* Calls BPF program right before an -++ * active connection is initialized -++ */ -++ BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB, /* Calls BPF program when an -++ * active connection is -++ * established -++ */ -++ BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB, /* Calls BPF program when a -++ * passive connection is -++ * established -++ */ -++ BPF_SOCK_OPS_NEEDS_ECN, /* If connection's congestion control -++ * needs ECN -++ */ -++ BPF_SOCK_OPS_BASE_RTT, /* Get base RTT. The correct value is -++ * based on the path and may be -++ * dependent on the congestion control -++ * algorithm. In general it indicates -++ * a congestion threshold. RTTs above -++ * this indicate congestion -++ */ -++ BPF_SOCK_OPS_RTO_CB, /* Called when an RTO has triggered. -++ * Arg1: value of icsk_retransmits -++ * Arg2: value of icsk_rto -++ * Arg3: whether RTO has expired -++ */ -++ BPF_SOCK_OPS_RETRANS_CB, /* Called when skb is retransmitted. -++ * Arg1: sequence number of 1st byte -++ * Arg2: # segments -++ * Arg3: return value of -++ * tcp_transmit_skb (0 => success) -++ */ -++ BPF_SOCK_OPS_STATE_CB, /* Called when TCP changes state. -++ * Arg1: old_state -++ * Arg2: new_state -++ */ -++ BPF_SOCK_OPS_TCP_LISTEN_CB, /* Called on listen(2), right after -++ * socket transition to LISTEN state. -++ */ -++ BPF_SOCK_OPS_RTT_CB, /* Called on every RTT. -++ */ -++}; -++ -++/* List of TCP states. There is a build check in net/ipv4/tcp.c to detect -++ * changes between the TCP and BPF versions. Ideally this should never happen. -++ * If it does, we need to add code to convert them before calling -++ * the BPF sock_ops function. -++ */ -++enum { -++ BPF_TCP_ESTABLISHED = 1, -++ BPF_TCP_SYN_SENT, -++ BPF_TCP_SYN_RECV, -++ BPF_TCP_FIN_WAIT1, -++ BPF_TCP_FIN_WAIT2, -++ BPF_TCP_TIME_WAIT, -++ BPF_TCP_CLOSE, -++ BPF_TCP_CLOSE_WAIT, -++ BPF_TCP_LAST_ACK, -++ BPF_TCP_LISTEN, -++ BPF_TCP_CLOSING, /* Now a valid state */ -++ BPF_TCP_NEW_SYN_RECV, -++ -++ BPF_TCP_MAX_STATES /* Leave at the end! */ -++}; -++ -++#define TCP_BPF_IW 1001 /* Set TCP initial congestion window */ -++#define TCP_BPF_SNDCWND_CLAMP 1002 /* Set sndcwnd_clamp */ -++ -++struct bpf_perf_event_value { -++ __u64 counter; -++ __u64 enabled; -++ __u64 running; -++}; -++ -++#define BPF_DEVCG_ACC_MKNOD (1ULL << 0) -++#define BPF_DEVCG_ACC_READ (1ULL << 1) -++#define BPF_DEVCG_ACC_WRITE (1ULL << 2) -++ -++#define BPF_DEVCG_DEV_BLOCK (1ULL << 0) -++#define BPF_DEVCG_DEV_CHAR (1ULL << 1) -++ -++struct bpf_cgroup_dev_ctx { -++ /* access_type encoded as (BPF_DEVCG_ACC_* << 16) | BPF_DEVCG_DEV_* */ -++ __u32 access_type; -++ __u32 major; -++ __u32 minor; -++}; -++ -++struct bpf_raw_tracepoint_args { -++ __u64 args[0]; -++}; -++ -++/* DIRECT: Skip the FIB rules and go to FIB table associated with device -++ * OUTPUT: Do lookup from egress perspective; default is ingress -++ */ -++#define BPF_FIB_LOOKUP_DIRECT (1U << 0) -++#define BPF_FIB_LOOKUP_OUTPUT (1U << 1) -++ -++enum { -++ BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */ -++ BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */ -++ BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */ -++ BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */ -++ BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */ -++ BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */ -++ BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ -++ BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ -++ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ -++}; -++ -++struct bpf_fib_lookup { -++ /* input: network family for lookup (AF_INET, AF_INET6) -++ * output: network family of egress nexthop -++ */ -++ __u8 family; -++ -++ /* set if lookup is to consider L4 data - e.g., FIB rules */ -++ __u8 l4_protocol; -++ __be16 sport; -++ __be16 dport; -++ -++ /* total length of packet from network header - used for MTU check */ -++ __u16 tot_len; -++ -++ /* input: L3 device index for lookup -++ * output: device index from FIB lookup -++ */ -++ __u32 ifindex; -++ -++ union { -++ /* inputs to lookup */ -++ __u8 tos; /* AF_INET */ -++ __be32 flowinfo; /* AF_INET6, flow_label + priority */ -++ -++ /* output: metric of fib result (IPv4/IPv6 only) */ -++ __u32 rt_metric; -++ }; -++ -++ union { -++ __be32 ipv4_src; -++ __u32 ipv6_src[4]; /* in6_addr; network order */ -++ }; -++ -++ /* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in -++ * network header. output: bpf_fib_lookup sets to gateway address -++ * if FIB lookup returns gateway route -++ */ -++ union { -++ __be32 ipv4_dst; -++ __u32 ipv6_dst[4]; /* in6_addr; network order */ -++ }; -++ -++ /* output */ -++ __be16 h_vlan_proto; -++ __be16 h_vlan_TCI; -++ __u8 smac[6]; /* ETH_ALEN */ -++ __u8 dmac[6]; /* ETH_ALEN */ -++}; -++ -++enum bpf_task_fd_type { -++ BPF_FD_TYPE_RAW_TRACEPOINT, /* tp name */ -++ BPF_FD_TYPE_TRACEPOINT, /* tp name */ -++ BPF_FD_TYPE_KPROBE, /* (symbol + offset) or addr */ -++ BPF_FD_TYPE_KRETPROBE, /* (symbol + offset) or addr */ -++ BPF_FD_TYPE_UPROBE, /* filename + offset */ -++ BPF_FD_TYPE_URETPROBE, /* filename + offset */ -++}; -++ -++#define BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG (1U << 0) -++#define BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL (1U << 1) -++#define BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP (1U << 2) -++ -++struct bpf_flow_keys { -++ __u16 nhoff; -++ __u16 thoff; -++ __u16 addr_proto; /* ETH_P_* of valid addrs */ -++ __u8 is_frag; -++ __u8 is_first_frag; -++ __u8 is_encap; -++ __u8 ip_proto; -++ __be16 n_proto; -++ __be16 sport; -++ __be16 dport; -++ union { -++ struct { -++ __be32 ipv4_src; -++ __be32 ipv4_dst; -++ }; -++ struct { -++ __u32 ipv6_src[4]; /* in6_addr; network order */ -++ __u32 ipv6_dst[4]; /* in6_addr; network order */ -++ }; -++ }; -++ __u32 flags; -++ __be32 flow_label; -++}; -++ -++struct bpf_func_info { -++ __u32 insn_off; -++ __u32 type_id; -++}; -++ -++#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10) -++#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff) -++ -++struct bpf_line_info { -++ __u32 insn_off; -++ __u32 file_name_off; -++ __u32 line_off; -++ __u32 line_col; -++}; -++ -++struct bpf_spin_lock { -++ __u32 val; -++}; -++ -++struct bpf_sysctl { -++ __u32 write; /* Sysctl is being read (= 0) or written (= 1). -++ * Allows 1,2,4-byte read, but no write. -++ */ -++ __u32 file_pos; /* Sysctl file position to read from, write to. -++ * Allows 1,2,4-byte read an 4-byte write. -++ */ -++}; -++ -++struct bpf_sockopt { -++ __bpf_md_ptr(struct bpf_sock *, sk); -++ __bpf_md_ptr(void *, optval); -++ __bpf_md_ptr(void *, optval_end); -++ -++ __s32 level; -++ __s32 optname; -++ __s32 optlen; -++ __s32 retval; -+ }; -+ -+ #endif /* _UAPI__LINUX_BPF_H__ */ -+--- /dev/null -++++ b/include/uapi/linux/bpfilter.h -+@@ -0,0 +1,21 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -++#ifndef _UAPI_LINUX_BPFILTER_H -++#define _UAPI_LINUX_BPFILTER_H -++ -++#include -++ -++enum { -++ BPFILTER_IPT_SO_SET_REPLACE = 64, -++ BPFILTER_IPT_SO_SET_ADD_COUNTERS = 65, -++ BPFILTER_IPT_SET_MAX, -++}; -++ -++enum { -++ BPFILTER_IPT_SO_GET_INFO = 64, -++ BPFILTER_IPT_SO_GET_ENTRIES = 65, -++ BPFILTER_IPT_SO_GET_REVISION_MATCH = 66, -++ BPFILTER_IPT_SO_GET_REVISION_TARGET = 67, -++ BPFILTER_IPT_GET_MAX, -++}; -++ -++#endif /* _UAPI_LINUX_BPFILTER_H */ -+--- /dev/null -++++ b/include/uapi/linux/bpf_perf_event.h -+@@ -0,0 +1,19 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -++/* Copyright (c) 2016 Facebook -++ * -++ * This program is free software; you can redistribute it and/or -++ * modify it under the terms of version 2 of the GNU General Public -++ * License as published by the Free Software Foundation. -++ */ -++#ifndef _UAPI__LINUX_BPF_PERF_EVENT_H__ -++#define _UAPI__LINUX_BPF_PERF_EVENT_H__ -++ -++#include -++ -++struct bpf_perf_event_data { -++ bpf_user_pt_regs_t regs; -++ __u64 sample_period; -++ __u64 addr; -++}; -++ -++#endif /* _UAPI__LINUX_BPF_PERF_EVENT_H__ */ -+--- /dev/null -++++ b/include/uapi/linux/btf.h -+@@ -0,0 +1,165 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -++/* Copyright (c) 2018 Facebook */ -++#ifndef _UAPI__LINUX_BTF_H__ -++#define _UAPI__LINUX_BTF_H__ -++ -++#include -++ -++#define BTF_MAGIC 0xeB9F -++#define BTF_VERSION 1 -++ -++struct btf_header { -++ __u16 magic; -++ __u8 version; -++ __u8 flags; -++ __u32 hdr_len; -++ -++ /* All offsets are in bytes relative to the end of this header */ -++ __u32 type_off; /* offset of type section */ -++ __u32 type_len; /* length of type section */ -++ __u32 str_off; /* offset of string section */ -++ __u32 str_len; /* length of string section */ -++}; -++ -++/* Max # of type identifier */ -++#define BTF_MAX_TYPE 0x000fffff -++/* Max offset into the string section */ -++#define BTF_MAX_NAME_OFFSET 0x00ffffff -++/* Max # of struct/union/enum members or func args */ -++#define BTF_MAX_VLEN 0xffff -++ -++struct btf_type { -++ __u32 name_off; -++ /* "info" bits arrangement -++ * bits 0-15: vlen (e.g. # of struct's members) -++ * bits 16-23: unused -++ * bits 24-27: kind (e.g. int, ptr, array...etc) -++ * bits 28-30: unused -++ * bit 31: kind_flag, currently used by -++ * struct, union and fwd -++ */ -++ __u32 info; -++ /* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC. -++ * "size" tells the size of the type it is describing. -++ * -++ * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, -++ * FUNC, FUNC_PROTO and VAR. -++ * "type" is a type_id referring to another type. -++ */ -++ union { -++ __u32 size; -++ __u32 type; -++ }; -++}; -++ -++#define BTF_INFO_KIND(info) (((info) >> 24) & 0x0f) -++#define BTF_INFO_VLEN(info) ((info) & 0xffff) -++#define BTF_INFO_KFLAG(info) ((info) >> 31) -++ -++#define BTF_KIND_UNKN 0 /* Unknown */ -++#define BTF_KIND_INT 1 /* Integer */ -++#define BTF_KIND_PTR 2 /* Pointer */ -++#define BTF_KIND_ARRAY 3 /* Array */ -++#define BTF_KIND_STRUCT 4 /* Struct */ -++#define BTF_KIND_UNION 5 /* Union */ -++#define BTF_KIND_ENUM 6 /* Enumeration */ -++#define BTF_KIND_FWD 7 /* Forward */ -++#define BTF_KIND_TYPEDEF 8 /* Typedef */ -++#define BTF_KIND_VOLATILE 9 /* Volatile */ -++#define BTF_KIND_CONST 10 /* Const */ -++#define BTF_KIND_RESTRICT 11 /* Restrict */ -++#define BTF_KIND_FUNC 12 /* Function */ -++#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ -++#define BTF_KIND_VAR 14 /* Variable */ -++#define BTF_KIND_DATASEC 15 /* Section */ -++#define BTF_KIND_MAX BTF_KIND_DATASEC -++#define NR_BTF_KINDS (BTF_KIND_MAX + 1) -++ -++/* For some specific BTF_KIND, "struct btf_type" is immediately -++ * followed by extra data. -++ */ -++ -++/* BTF_KIND_INT is followed by a u32 and the following -++ * is the 32 bits arrangement: -++ */ -++#define BTF_INT_ENCODING(VAL) (((VAL) & 0x0f000000) >> 24) -++#define BTF_INT_OFFSET(VAL) (((VAL) & 0x00ff0000) >> 16) -++#define BTF_INT_BITS(VAL) ((VAL) & 0x000000ff) -++ -++/* Attributes stored in the BTF_INT_ENCODING */ -++#define BTF_INT_SIGNED (1 << 0) -++#define BTF_INT_CHAR (1 << 1) -++#define BTF_INT_BOOL (1 << 2) -++ -++/* BTF_KIND_ENUM is followed by multiple "struct btf_enum". -++ * The exact number of btf_enum is stored in the vlen (of the -++ * info in "struct btf_type"). -++ */ -++struct btf_enum { -++ __u32 name_off; -++ __s32 val; -++}; -++ -++/* BTF_KIND_ARRAY is followed by one "struct btf_array" */ -++struct btf_array { -++ __u32 type; -++ __u32 index_type; -++ __u32 nelems; -++}; -++ -++/* BTF_KIND_STRUCT and BTF_KIND_UNION are followed -++ * by multiple "struct btf_member". The exact number -++ * of btf_member is stored in the vlen (of the info in -++ * "struct btf_type"). -++ */ -++struct btf_member { -++ __u32 name_off; -++ __u32 type; -++ /* If the type info kind_flag is set, the btf_member offset -++ * contains both member bitfield size and bit offset. The -++ * bitfield size is set for bitfield members. If the type -++ * info kind_flag is not set, the offset contains only bit -++ * offset. -++ */ -++ __u32 offset; -++}; -++ -++/* If the struct/union type info kind_flag is set, the -++ * following two macros are used to access bitfield_size -++ * and bit_offset from btf_member.offset. -++ */ -++#define BTF_MEMBER_BITFIELD_SIZE(val) ((val) >> 24) -++#define BTF_MEMBER_BIT_OFFSET(val) ((val) & 0xffffff) -++ -++/* BTF_KIND_FUNC_PROTO is followed by multiple "struct btf_param". -++ * The exact number of btf_param is stored in the vlen (of the -++ * info in "struct btf_type"). -++ */ -++struct btf_param { -++ __u32 name_off; -++ __u32 type; -++}; -++ -++enum { -++ BTF_VAR_STATIC = 0, -++ BTF_VAR_GLOBAL_ALLOCATED, -++}; -++ -++/* BTF_KIND_VAR is followed by a single "struct btf_var" to describe -++ * additional information related to the variable such as its linkage. -++ */ -++struct btf_var { -++ __u32 linkage; -++}; -++ -++/* BTF_KIND_DATASEC is followed by multiple "struct btf_var_secinfo" -++ * to describe all BTF_KIND_VAR types it contains along with it's -++ * in-section offset as well as size. -++ */ -++struct btf_var_secinfo { -++ __u32 type; -++ __u32 offset; -++ __u32 size; -++}; -++ -++#endif /* _UAPI__LINUX_BTF_H__ */ -+--- a/kernel/bpf/arraymap.c -++++ b/kernel/bpf/arraymap.c -+@@ -1,78 +1,141 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+- * -+- * This program is distributed in the hope that it will be useful, but -+- * WITHOUT ANY WARRANTY; without even the implied warranty of -+- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+- * General Public License for more details. -++ * Copyright (c) 2016,2017 Facebook -+ */ -+ #include -++#include -+ #include -+-#include -+ #include -+ #include -+ #include -+ #include -++#include -++ -++#include "map_in_map.h" -++ -++#define ARRAY_CREATE_FLAG_MASK \ -++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) -++ -++static void bpf_array_free_percpu(struct bpf_array *array) -++{ -++ int i; -++ -++ for (i = 0; i < array->map.max_entries; i++) { -++ free_percpu(array->pptrs[i]); -++ cond_resched(); -++ } -++} -++ -++static int bpf_array_alloc_percpu(struct bpf_array *array) -++{ -++ void __percpu *ptr; -++ int i; -++ -++ for (i = 0; i < array->map.max_entries; i++) { -++ ptr = __alloc_percpu_gfp(array->elem_size, 8, -++ GFP_USER | __GFP_NOWARN); -++ if (!ptr) { -++ bpf_array_free_percpu(array); -++ return -ENOMEM; -++ } -++ array->pptrs[i] = ptr; -++ cond_resched(); -++ } -++ -++ return 0; -++} -+ -+ /* Called from syscall */ -+-static struct bpf_map *array_map_alloc(union bpf_attr *attr) -++int array_map_alloc_check(union bpf_attr *attr) -+ { -+- struct bpf_array *array; -+- u32 elem_size, array_size; -+- u32 index_mask, max_entries; -+- bool unpriv = !capable(CAP_SYS_ADMIN); -++ bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; -++ int numa_node = bpf_map_attr_numa_node(attr); -+ -+ /* check sanity of attributes */ -+ if (attr->max_entries == 0 || attr->key_size != 4 || -+- attr->value_size == 0) -+- return ERR_PTR(-EINVAL); -++ attr->value_size == 0 || -++ attr->map_flags & ~ARRAY_CREATE_FLAG_MASK || -++ !bpf_map_flags_access_ok(attr->map_flags) || -++ (percpu && numa_node != NUMA_NO_NODE)) -++ return -EINVAL; -+ -+- if (attr->value_size >= 1 << (KMALLOC_SHIFT_MAX - 1)) -++ if (attr->value_size > KMALLOC_MAX_SIZE) -+ /* if value_size is bigger, the user space won't be able to -+ * access the elements. -+ */ -+- return ERR_PTR(-E2BIG); -++ return -E2BIG; -++ -++ return 0; -++} -++ -++static struct bpf_map *array_map_alloc(union bpf_attr *attr) -++{ -++ bool percpu = attr->map_type == BPF_MAP_TYPE_PERCPU_ARRAY; -++ int ret, numa_node = bpf_map_attr_numa_node(attr); -++ u32 elem_size, index_mask, max_entries; -++ bool unpriv = !capable(CAP_SYS_ADMIN); -++ u64 cost, array_size, mask64; -++ struct bpf_map_memory mem; -++ struct bpf_array *array; -+ -+ elem_size = round_up(attr->value_size, 8); -+ -+ max_entries = attr->max_entries; -+- index_mask = roundup_pow_of_two(max_entries) - 1; -+ -+- if (unpriv) -++ /* On 32 bit archs roundup_pow_of_two() with max_entries that has -++ * upper most bit set in u32 space is undefined behavior due to -++ * resulting 1U << 32, so do it manually here in u64 space. -++ */ -++ mask64 = fls_long(max_entries - 1); -++ mask64 = 1ULL << mask64; -++ mask64 -= 1; -++ -++ index_mask = mask64; -++ if (unpriv) { -+ /* round up array size to nearest power of 2, -+ * since cpu will speculate within index_mask limits -+ */ -+ max_entries = index_mask + 1; -++ /* Check for overflows. */ -++ if (max_entries < attr->max_entries) -++ return ERR_PTR(-E2BIG); -++ } -+ -+- -+- /* check round_up into zero and u32 overflow */ -+- if (elem_size == 0 || -+- attr->max_entries > (U32_MAX - PAGE_SIZE - sizeof(*array)) / elem_size) -+- return ERR_PTR(-ENOMEM); -+- -+- array_size = sizeof(*array) + max_entries * elem_size; -++ array_size = sizeof(*array); -++ if (percpu) -++ array_size += (u64) max_entries * sizeof(void *); -++ else -++ array_size += (u64) max_entries * elem_size; -++ -++ /* make sure there is no u32 overflow later in round_up() */ -++ cost = array_size; -++ if (percpu) -++ cost += (u64)attr->max_entries * elem_size * num_possible_cpus(); -++ -++ ret = bpf_map_charge_init(&mem, cost); -++ if (ret < 0) -++ return ERR_PTR(ret); -+ -+ /* allocate all map elements and zero-initialize them */ -+- array = kzalloc(array_size, GFP_USER | __GFP_NOWARN); -++ array = bpf_map_area_alloc(array_size, numa_node); -+ if (!array) { -+- array = vzalloc(array_size); -+- if (!array) -+- return ERR_PTR(-ENOMEM); -++ bpf_map_charge_finish(&mem); -++ return ERR_PTR(-ENOMEM); -+ } -+- -+ array->index_mask = index_mask; -+ array->map.unpriv_array = unpriv; -+ -+ /* copy mandatory map attributes */ -+- array->map.key_size = attr->key_size; -+- array->map.value_size = attr->value_size; -+- array->map.max_entries = attr->max_entries; -+- array->map.pages = round_up(array_size, PAGE_SIZE) >> PAGE_SHIFT; -++ bpf_map_init_from_attr(&array->map, attr); -++ bpf_map_charge_move(&array->map.memory, &mem); -+ array->elem_size = elem_size; -+ -++ if (percpu && bpf_array_alloc_percpu(array)) { -++ bpf_map_charge_finish(&array->map.memory); -++ bpf_map_area_free(array); -++ return ERR_PTR(-ENOMEM); -++ } -++ -+ return &array->map; -+ } -+ -+@@ -82,17 +145,115 @@ static void *array_map_lookup_elem(struc -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ u32 index = *(u32 *)key; -+ -+- if (index >= array->map.max_entries) -++ if (unlikely(index >= array->map.max_entries)) -+ return NULL; -+ -+ return array->value + array->elem_size * (index & array->index_mask); -+ } -+ -++static int array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, -++ u32 off) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ -++ if (map->max_entries != 1) -++ return -ENOTSUPP; -++ if (off >= map->value_size) -++ return -EINVAL; -++ -++ *imm = (unsigned long)array->value; -++ return 0; -++} -++ -++static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, -++ u32 *off) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u64 base = (unsigned long)array->value; -++ u64 range = array->elem_size; -++ -++ if (map->max_entries != 1) -++ return -ENOTSUPP; -++ if (imm < base || imm >= base + range) -++ return -ENOENT; -++ -++ *off = imm - base; -++ return 0; -++} -++ -++/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */ -++static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ struct bpf_insn *insn = insn_buf; -++ u32 elem_size = round_up(map->value_size, 8); -++ const int ret = BPF_REG_0; -++ const int map_ptr = BPF_REG_1; -++ const int index = BPF_REG_2; -++ -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); -++ *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); -++ if (map->unpriv_array) { -++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 4); -++ *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); -++ } else { -++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 3); -++ } -++ -++ if (is_power_of_2(elem_size)) { -++ *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); -++ } else { -++ *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); -++ } -++ *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); -++ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); -++ *insn++ = BPF_MOV64_IMM(ret, 0); -++ return insn - insn_buf; -++} -++ -++/* Called from eBPF program */ -++static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u32 index = *(u32 *)key; -++ -++ if (unlikely(index >= array->map.max_entries)) -++ return NULL; -++ -++ return this_cpu_ptr(array->pptrs[index & array->index_mask]); -++} -++ -++int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u32 index = *(u32 *)key; -++ void __percpu *pptr; -++ int cpu, off = 0; -++ u32 size; -++ -++ if (unlikely(index >= array->map.max_entries)) -++ return -ENOENT; -++ -++ /* per_cpu areas are zero-filled and bpf programs can only -++ * access 'value_size' of them, so copying rounded areas -++ * will not leak any kernel data -++ */ -++ size = round_up(map->value_size, 8); -++ rcu_read_lock(); -++ pptr = array->pptrs[index & array->index_mask]; -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size); -++ off += size; -++ } -++ rcu_read_unlock(); -++ return 0; -++} -++ -+ /* Called from syscall */ -+ static int array_map_get_next_key(struct bpf_map *map, void *key, void *next_key) -+ { -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+- u32 index = *(u32 *)key; -++ u32 index = key ? *(u32 *)key : U32_MAX; -+ u32 *next = (u32 *)next_key; -+ -+ if (index >= array->map.max_entries) { -+@@ -113,22 +274,73 @@ static int array_map_update_elem(struct -+ { -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ u32 index = *(u32 *)key; -++ char *val; -+ -+- if (map_flags > BPF_EXIST) -++ if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) -+ /* unknown flags */ -+ return -EINVAL; -+ -+- if (index >= array->map.max_entries) -++ if (unlikely(index >= array->map.max_entries)) -++ /* all elements were pre-allocated, cannot insert a new one */ -++ return -E2BIG; -++ -++ if (unlikely(map_flags & BPF_NOEXIST)) -++ /* all elements already exist */ -++ return -EEXIST; -++ -++ if (unlikely((map_flags & BPF_F_LOCK) && -++ !map_value_has_spin_lock(map))) -++ return -EINVAL; -++ -++ if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { -++ memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), -++ value, map->value_size); -++ } else { -++ val = array->value + -++ array->elem_size * (index & array->index_mask); -++ if (map_flags & BPF_F_LOCK) -++ copy_map_value_locked(map, val, value, false); -++ else -++ copy_map_value(map, val, value); -++ } -++ return 0; -++} -++ -++int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u32 index = *(u32 *)key; -++ void __percpu *pptr; -++ int cpu, off = 0; -++ u32 size; -++ -++ if (unlikely(map_flags > BPF_EXIST)) -++ /* unknown flags */ -++ return -EINVAL; -++ -++ if (unlikely(index >= array->map.max_entries)) -+ /* all elements were pre-allocated, cannot insert a new one */ -+ return -E2BIG; -+ -+- if (map_flags == BPF_NOEXIST) -++ if (unlikely(map_flags == BPF_NOEXIST)) -+ /* all elements already exist */ -+ return -EEXIST; -+ -+- memcpy(array->value + -+- array->elem_size * (index & array->index_mask), -+- value, map->value_size); -++ /* the user space will provide round_up(value_size, 8) bytes that -++ * will be copied into per-cpu area. bpf programs can only access -++ * value_size of it. During lookup the same extra bytes will be -++ * returned or zeros which were zero-filled by percpu_alloc, -++ * so no kernel data leaks possible -++ */ -++ size = round_up(map->value_size, 8); -++ rcu_read_lock(); -++ pptr = array->pptrs[index & array->index_mask]; -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size); -++ off += size; -++ } -++ rcu_read_unlock(); -+ return 0; -+ } -+ -+@@ -150,36 +362,124 @@ static void array_map_free(struct bpf_ma -+ */ -+ synchronize_rcu(); -+ -+- kvfree(array); -++ if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) -++ bpf_array_free_percpu(array); -++ -++ bpf_map_area_free(array); -+ } -+ -+-static const struct bpf_map_ops array_ops = { -++static void array_map_seq_show_elem(struct bpf_map *map, void *key, -++ struct seq_file *m) -++{ -++ void *value; -++ -++ rcu_read_lock(); -++ -++ value = array_map_lookup_elem(map, key); -++ if (!value) { -++ rcu_read_unlock(); -++ return; -++ } -++ -++ if (map->btf_key_type_id) -++ seq_printf(m, "%u: ", *(u32 *)key); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); -++ seq_puts(m, "\n"); -++ -++ rcu_read_unlock(); -++} -++ -++static void percpu_array_map_seq_show_elem(struct bpf_map *map, void *key, -++ struct seq_file *m) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u32 index = *(u32 *)key; -++ void __percpu *pptr; -++ int cpu; -++ -++ rcu_read_lock(); -++ -++ seq_printf(m, "%u: {\n", *(u32 *)key); -++ pptr = array->pptrs[index & array->index_mask]; -++ for_each_possible_cpu(cpu) { -++ seq_printf(m, "\tcpu%d: ", cpu); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, -++ per_cpu_ptr(pptr, cpu), m); -++ seq_puts(m, "\n"); -++ } -++ seq_puts(m, "}\n"); -++ -++ rcu_read_unlock(); -++} -++ -++static int array_map_check_btf(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type) -++{ -++ u32 int_data; -++ -++ /* One exception for keyless BTF: .bss/.data/.rodata map */ -++ if (btf_type_is_void(key_type)) { -++ if (map->map_type != BPF_MAP_TYPE_ARRAY || -++ map->max_entries != 1) -++ return -EINVAL; -++ -++ if (BTF_INFO_KIND(value_type->info) != BTF_KIND_DATASEC) -++ return -EINVAL; -++ -++ return 0; -++ } -++ -++ if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT) -++ return -EINVAL; -++ -++ int_data = *(u32 *)(key_type + 1); -++ /* bpf array can only take a u32 key. This check makes sure -++ * that the btf matches the attr used during map_create. -++ */ -++ if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data)) -++ return -EINVAL; -++ -++ return 0; -++} -++ -++const struct bpf_map_ops array_map_ops = { -++ .map_alloc_check = array_map_alloc_check, -+ .map_alloc = array_map_alloc, -+ .map_free = array_map_free, -+ .map_get_next_key = array_map_get_next_key, -+ .map_lookup_elem = array_map_lookup_elem, -+ .map_update_elem = array_map_update_elem, -+ .map_delete_elem = array_map_delete_elem, -++ .map_gen_lookup = array_map_gen_lookup, -++ .map_direct_value_addr = array_map_direct_value_addr, -++ .map_direct_value_meta = array_map_direct_value_meta, -++ .map_seq_show_elem = array_map_seq_show_elem, -++ .map_check_btf = array_map_check_btf, -+ }; -+ -+-static struct bpf_map_type_list array_type __read_mostly = { -+- .ops = &array_ops, -+- .type = BPF_MAP_TYPE_ARRAY, -++const struct bpf_map_ops percpu_array_map_ops = { -++ .map_alloc_check = array_map_alloc_check, -++ .map_alloc = array_map_alloc, -++ .map_free = array_map_free, -++ .map_get_next_key = array_map_get_next_key, -++ .map_lookup_elem = percpu_array_map_lookup_elem, -++ .map_update_elem = array_map_update_elem, -++ .map_delete_elem = array_map_delete_elem, -++ .map_seq_show_elem = percpu_array_map_seq_show_elem, -++ .map_check_btf = array_map_check_btf, -+ }; -+ -+-static int __init register_array_map(void) -+-{ -+- bpf_register_map_type(&array_type); -+- return 0; -+-} -+-late_initcall(register_array_map); -+- -+-static struct bpf_map *fd_array_map_alloc(union bpf_attr *attr) -++static int fd_array_map_alloc_check(union bpf_attr *attr) -+ { -+ /* only file descriptors can be stored in this type of map */ -+ if (attr->value_size != sizeof(u32)) -+- return ERR_PTR(-EINVAL); -+- return array_map_alloc(attr); -++ return -EINVAL; -++ /* Program read-only/write-only not supported for special maps yet. */ -++ if (attr->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) -++ return -EINVAL; -++ return array_map_alloc_check(attr); -+ } -+ -+ static void fd_array_map_free(struct bpf_map *map) -+@@ -192,17 +492,38 @@ static void fd_array_map_free(struct bpf -+ /* make sure it's empty */ -+ for (i = 0; i < array->map.max_entries; i++) -+ BUG_ON(array->ptrs[i] != NULL); -+- kvfree(array); -++ -++ bpf_map_area_free(array); -+ } -+ -+ static void *fd_array_map_lookup_elem(struct bpf_map *map, void *key) -+ { -+- return NULL; -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++/* only called from syscall */ -++int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) -++{ -++ void **elem, *ptr; -++ int ret = 0; -++ -++ if (!map->ops->map_fd_sys_lookup_elem) -++ return -ENOTSUPP; -++ -++ rcu_read_lock(); -++ elem = array_map_lookup_elem(map, key); -++ if (elem && (ptr = READ_ONCE(*elem))) -++ *value = map->ops->map_fd_sys_lookup_elem(ptr); -++ else -++ ret = -ENOENT; -++ rcu_read_unlock(); -++ -++ return ret; -+ } -+ -+ /* only called from syscall */ -+-static int fd_array_map_update_elem(struct bpf_map *map, void *key, -+- void *value, u64 map_flags) -++int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, -++ void *key, void *value, u64 map_flags) -+ { -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ void *new_ptr, *old_ptr; -+@@ -215,7 +536,7 @@ static int fd_array_map_update_elem(stru -+ return -E2BIG; -+ -+ ufd = *(u32 *)value; -+- new_ptr = map->ops->map_fd_get_ptr(map, ufd); -++ new_ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); -+ if (IS_ERR(new_ptr)) -+ return PTR_ERR(new_ptr); -+ -+@@ -244,10 +565,12 @@ static int fd_array_map_delete_elem(stru -+ } -+ } -+ -+-static void *prog_fd_array_get_ptr(struct bpf_map *map, int fd) -++static void *prog_fd_array_get_ptr(struct bpf_map *map, -++ struct file *map_file, int fd) -+ { -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ struct bpf_prog *prog = bpf_prog_get(fd); -++ -+ if (IS_ERR(prog)) -+ return prog; -+ -+@@ -255,18 +578,22 @@ static void *prog_fd_array_get_ptr(struc -+ bpf_prog_put(prog); -+ return ERR_PTR(-EINVAL); -+ } -++ -+ return prog; -+ } -+ -+ static void prog_fd_array_put_ptr(void *ptr) -+ { -+- struct bpf_prog *prog = ptr; -++ bpf_prog_put(ptr); -++} -+ -+- bpf_prog_put_rcu(prog); -++static u32 prog_fd_array_sys_lookup_elem(void *ptr) -++{ -++ return ((struct bpf_prog *)ptr)->aux->id; -+ } -+ -+ /* decrement refcnt of all bpf_progs that are stored in this map */ -+-void bpf_fd_array_map_clear(struct bpf_map *map) -++static void bpf_fd_array_map_clear(struct bpf_map *map) -+ { -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ int i; -+@@ -275,91 +602,208 @@ void bpf_fd_array_map_clear(struct bpf_m -+ fd_array_map_delete_elem(map, &i); -+ } -+ -+-static const struct bpf_map_ops prog_array_ops = { -+- .map_alloc = fd_array_map_alloc, -++static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, -++ struct seq_file *m) -++{ -++ void **elem, *ptr; -++ u32 prog_id; -++ -++ rcu_read_lock(); -++ -++ elem = array_map_lookup_elem(map, key); -++ if (elem) { -++ ptr = READ_ONCE(*elem); -++ if (ptr) { -++ seq_printf(m, "%u: ", *(u32 *)key); -++ prog_id = prog_fd_array_sys_lookup_elem(ptr); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, -++ &prog_id, m); -++ seq_puts(m, "\n"); -++ } -++ } -++ -++ rcu_read_unlock(); -++} -++ -++const struct bpf_map_ops prog_array_map_ops = { -++ .map_alloc_check = fd_array_map_alloc_check, -++ .map_alloc = array_map_alloc, -+ .map_free = fd_array_map_free, -+ .map_get_next_key = array_map_get_next_key, -+ .map_lookup_elem = fd_array_map_lookup_elem, -+- .map_update_elem = fd_array_map_update_elem, -+ .map_delete_elem = fd_array_map_delete_elem, -+ .map_fd_get_ptr = prog_fd_array_get_ptr, -+ .map_fd_put_ptr = prog_fd_array_put_ptr, -++ .map_fd_sys_lookup_elem = prog_fd_array_sys_lookup_elem, -++ .map_release_uref = bpf_fd_array_map_clear, -++ .map_seq_show_elem = prog_array_map_seq_show_elem, -+ }; -+ -+-static struct bpf_map_type_list prog_array_type __read_mostly = { -+- .ops = &prog_array_ops, -+- .type = BPF_MAP_TYPE_PROG_ARRAY, -+-}; -++static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file, -++ struct file *map_file) -++{ -++ struct bpf_event_entry *ee; -++ -++ ee = kzalloc(sizeof(*ee), GFP_ATOMIC); -++ if (ee) { -++ ee->event = perf_file->private_data; -++ ee->perf_file = perf_file; -++ ee->map_file = map_file; -++ } -+ -+-static int __init register_prog_array_map(void) -++ return ee; -++} -++ -++static void __bpf_event_entry_free(struct rcu_head *rcu) -+ { -+- bpf_register_map_type(&prog_array_type); -+- return 0; -++ struct bpf_event_entry *ee; -++ -++ ee = container_of(rcu, struct bpf_event_entry, rcu); -++ fput(ee->perf_file); -++ kfree(ee); -+ } -+-late_initcall(register_prog_array_map); -+ -+-static void perf_event_array_map_free(struct bpf_map *map) -++static void bpf_event_entry_free_rcu(struct bpf_event_entry *ee) -+ { -+- bpf_fd_array_map_clear(map); -+- fd_array_map_free(map); -++ call_rcu(&ee->rcu, __bpf_event_entry_free); -+ } -+ -+-static void *perf_event_fd_array_get_ptr(struct bpf_map *map, int fd) -++static void *perf_event_fd_array_get_ptr(struct bpf_map *map, -++ struct file *map_file, int fd) -+ { -++ struct bpf_event_entry *ee; -+ struct perf_event *event; -+- const struct perf_event_attr *attr; -++ struct file *perf_file; -++ u64 value; -+ -+ event = perf_event_get(fd); -+ if (IS_ERR(event)) -+ return event; -+ -+- attr = perf_event_attrs(event); -+- if (IS_ERR(attr)) -+- goto err; -+- -+- if (attr->inherit) -+- goto err; -++ value = perf_event_read_local(event); -+ -+- if (attr->type == PERF_TYPE_RAW) -+- return event; -+- -+- if (attr->type == PERF_TYPE_HARDWARE) -+- return event; -++ ee = bpf_event_entry_gen(perf_file, map_file); -++ if (ee) -++ return ee; -+ -+- if (attr->type == PERF_TYPE_SOFTWARE && -+- attr->config == PERF_COUNT_SW_BPF_OUTPUT) -+- return event; -+-err: -+- perf_event_release_kernel(event); -+- return ERR_PTR(-EINVAL); -++ ee = ERR_PTR(-ENOMEM); -++ return ee; -+ } -+ -+ static void perf_event_fd_array_put_ptr(void *ptr) -+ { -+- struct perf_event *event = ptr; -++ bpf_event_entry_free_rcu(ptr); -++} -+ -+- perf_event_release_kernel(event); -++static void perf_event_fd_array_release(struct bpf_map *map, -++ struct file *map_file) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ struct bpf_event_entry *ee; -++ int i; -++ -++ rcu_read_lock(); -++ for (i = 0; i < array->map.max_entries; i++) { -++ ee = READ_ONCE(array->ptrs[i]); -++ if (ee && ee->map_file == map_file) -++ fd_array_map_delete_elem(map, &i); -++ } -++ rcu_read_unlock(); -+ } -+ -+-static const struct bpf_map_ops perf_event_array_ops = { -+- .map_alloc = fd_array_map_alloc, -+- .map_free = perf_event_array_map_free, -++const struct bpf_map_ops perf_event_array_map_ops = { -++ .map_alloc_check = fd_array_map_alloc_check, -++ .map_alloc = array_map_alloc, -++ .map_free = fd_array_map_free, -+ .map_get_next_key = array_map_get_next_key, -+ .map_lookup_elem = fd_array_map_lookup_elem, -+- .map_update_elem = fd_array_map_update_elem, -+ .map_delete_elem = fd_array_map_delete_elem, -+ .map_fd_get_ptr = perf_event_fd_array_get_ptr, -+ .map_fd_put_ptr = perf_event_fd_array_put_ptr, -++ .map_release = perf_event_fd_array_release, -++ .map_check_btf = map_check_no_btf, -+ }; -+ -+-static struct bpf_map_type_list perf_event_array_type __read_mostly = { -+- .ops = &perf_event_array_ops, -+- .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, -+-}; -++static struct bpf_map *array_of_map_alloc(union bpf_attr *attr) -++{ -++ struct bpf_map *map, *inner_map_meta; -++ -++ inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); -++ if (IS_ERR(inner_map_meta)) -++ return inner_map_meta; -++ -++ map = array_map_alloc(attr); -++ if (IS_ERR(map)) { -++ bpf_map_meta_free(inner_map_meta); -++ return map; -++ } -++ -++ map->inner_map_meta = inner_map_meta; -+ -+-static int __init register_perf_event_array_map(void) -++ return map; -++} -++ -++static void array_of_map_free(struct bpf_map *map) -+ { -+- bpf_register_map_type(&perf_event_array_type); -+- return 0; -++ /* map->inner_map_meta is only accessed by syscall which -++ * is protected by fdget/fdput. -++ */ -++ bpf_map_meta_free(map->inner_map_meta); -++ bpf_fd_array_map_clear(map); -++ fd_array_map_free(map); -+ } -+-late_initcall(register_perf_event_array_map); -++ -++static void *array_of_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_map **inner_map = array_map_lookup_elem(map, key); -++ -++ if (!inner_map) -++ return NULL; -++ -++ return READ_ONCE(*inner_map); -++} -++ -++static u32 array_of_map_gen_lookup(struct bpf_map *map, -++ struct bpf_insn *insn_buf) -++{ -++ struct bpf_array *array = container_of(map, struct bpf_array, map); -++ u32 elem_size = round_up(map->value_size, 8); -++ struct bpf_insn *insn = insn_buf; -++ const int ret = BPF_REG_0; -++ const int map_ptr = BPF_REG_1; -++ const int index = BPF_REG_2; -++ -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value)); -++ *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); -++ if (map->unpriv_array) { -++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 6); -++ *insn++ = BPF_ALU32_IMM(BPF_AND, ret, array->index_mask); -++ } else { -++ *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); -++ } -++ if (is_power_of_2(elem_size)) -++ *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(elem_size)); -++ else -++ *insn++ = BPF_ALU64_IMM(BPF_MUL, ret, elem_size); -++ *insn++ = BPF_ALU64_REG(BPF_ADD, ret, map_ptr); -++ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); -++ *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); -++ *insn++ = BPF_MOV64_IMM(ret, 0); -++ -++ return insn - insn_buf; -++} -++ -++const struct bpf_map_ops array_of_maps_map_ops = { -++ .map_alloc_check = fd_array_map_alloc_check, -++ .map_alloc = array_of_map_alloc, -++ .map_free = array_of_map_free, -++ .map_get_next_key = array_map_get_next_key, -++ .map_lookup_elem = array_of_map_lookup_elem, -++ .map_delete_elem = fd_array_map_delete_elem, -++ .map_fd_get_ptr = bpf_map_fd_get_ptr, -++ .map_fd_put_ptr = bpf_map_fd_put_ptr, -++ .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, -++ .map_gen_lookup = array_of_map_gen_lookup, -++ .map_check_btf = map_check_no_btf, -++}; -+--- /dev/null -++++ b/kernel/bpf/bpf_lru_list.c -+@@ -0,0 +1,695 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2016 Facebook -++ */ -++#include -++#include -++#include -++ -++#include "bpf_lru_list.h" -++ -++#define LOCAL_FREE_TARGET (128) -++#define LOCAL_NR_SCANS LOCAL_FREE_TARGET -++ -++#define PERCPU_FREE_TARGET (4) -++#define PERCPU_NR_SCANS PERCPU_FREE_TARGET -++ -++/* Helpers to get the local list index */ -++#define LOCAL_LIST_IDX(t) ((t) - BPF_LOCAL_LIST_T_OFFSET) -++#define LOCAL_FREE_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_FREE) -++#define LOCAL_PENDING_LIST_IDX LOCAL_LIST_IDX(BPF_LRU_LOCAL_LIST_T_PENDING) -++#define IS_LOCAL_LIST_TYPE(t) ((t) >= BPF_LOCAL_LIST_T_OFFSET) -++ -++static int get_next_cpu(int cpu) -++{ -++ cpu = cpumask_next(cpu, cpu_possible_mask); -++ if (cpu >= nr_cpu_ids) -++ cpu = cpumask_first(cpu_possible_mask); -++ return cpu; -++} -++ -++/* Local list helpers */ -++static struct list_head *local_free_list(struct bpf_lru_locallist *loc_l) -++{ -++ return &loc_l->lists[LOCAL_FREE_LIST_IDX]; -++} -++ -++static struct list_head *local_pending_list(struct bpf_lru_locallist *loc_l) -++{ -++ return &loc_l->lists[LOCAL_PENDING_LIST_IDX]; -++} -++ -++/* bpf_lru_node helpers */ -++static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node) -++{ -++ return node->ref; -++} -++ -++static void bpf_lru_list_count_inc(struct bpf_lru_list *l, -++ enum bpf_lru_list_type type) -++{ -++ if (type < NR_BPF_LRU_LIST_COUNT) -++ l->counts[type]++; -++} -++ -++static void bpf_lru_list_count_dec(struct bpf_lru_list *l, -++ enum bpf_lru_list_type type) -++{ -++ if (type < NR_BPF_LRU_LIST_COUNT) -++ l->counts[type]--; -++} -++ -++static void __bpf_lru_node_move_to_free(struct bpf_lru_list *l, -++ struct bpf_lru_node *node, -++ struct list_head *free_list, -++ enum bpf_lru_list_type tgt_free_type) -++{ -++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) -++ return; -++ -++ /* If the removing node is the next_inactive_rotation candidate, -++ * move the next_inactive_rotation pointer also. -++ */ -++ if (&node->list == l->next_inactive_rotation) -++ l->next_inactive_rotation = l->next_inactive_rotation->prev; -++ -++ bpf_lru_list_count_dec(l, node->type); -++ -++ node->type = tgt_free_type; -++ list_move(&node->list, free_list); -++} -++ -++/* Move nodes from local list to the LRU list */ -++static void __bpf_lru_node_move_in(struct bpf_lru_list *l, -++ struct bpf_lru_node *node, -++ enum bpf_lru_list_type tgt_type) -++{ -++ if (WARN_ON_ONCE(!IS_LOCAL_LIST_TYPE(node->type)) || -++ WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(tgt_type))) -++ return; -++ -++ bpf_lru_list_count_inc(l, tgt_type); -++ node->type = tgt_type; -++ node->ref = 0; -++ list_move(&node->list, &l->lists[tgt_type]); -++} -++ -++/* Move nodes between or within active and inactive list (like -++ * active to inactive, inactive to active or tail of active back to -++ * the head of active). -++ */ -++static void __bpf_lru_node_move(struct bpf_lru_list *l, -++ struct bpf_lru_node *node, -++ enum bpf_lru_list_type tgt_type) -++{ -++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)) || -++ WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(tgt_type))) -++ return; -++ -++ if (node->type != tgt_type) { -++ bpf_lru_list_count_dec(l, node->type); -++ bpf_lru_list_count_inc(l, tgt_type); -++ node->type = tgt_type; -++ } -++ node->ref = 0; -++ -++ /* If the moving node is the next_inactive_rotation candidate, -++ * move the next_inactive_rotation pointer also. -++ */ -++ if (&node->list == l->next_inactive_rotation) -++ l->next_inactive_rotation = l->next_inactive_rotation->prev; -++ -++ list_move(&node->list, &l->lists[tgt_type]); -++} -++ -++static bool bpf_lru_list_inactive_low(const struct bpf_lru_list *l) -++{ -++ return l->counts[BPF_LRU_LIST_T_INACTIVE] < -++ l->counts[BPF_LRU_LIST_T_ACTIVE]; -++} -++ -++/* Rotate the active list: -++ * 1. Start from tail -++ * 2. If the node has the ref bit set, it will be rotated -++ * back to the head of active list with the ref bit cleared. -++ * Give this node one more chance to survive in the active list. -++ * 3. If the ref bit is not set, move it to the head of the -++ * inactive list. -++ * 4. It will at most scan nr_scans nodes -++ */ -++static void __bpf_lru_list_rotate_active(struct bpf_lru *lru, -++ struct bpf_lru_list *l) -++{ -++ struct list_head *active = &l->lists[BPF_LRU_LIST_T_ACTIVE]; -++ struct bpf_lru_node *node, *tmp_node, *first_node; -++ unsigned int i = 0; -++ -++ first_node = list_first_entry(active, struct bpf_lru_node, list); -++ list_for_each_entry_safe_reverse(node, tmp_node, active, list) { -++ if (bpf_lru_node_is_ref(node)) -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); -++ else -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); -++ -++ if (++i == lru->nr_scans || node == first_node) -++ break; -++ } -++} -++ -++/* Rotate the inactive list. It starts from the next_inactive_rotation -++ * 1. If the node has ref bit set, it will be moved to the head -++ * of active list with the ref bit cleared. -++ * 2. If the node does not have ref bit set, it will leave it -++ * at its current location (i.e. do nothing) so that it can -++ * be considered during the next inactive_shrink. -++ * 3. It will at most scan nr_scans nodes -++ */ -++static void __bpf_lru_list_rotate_inactive(struct bpf_lru *lru, -++ struct bpf_lru_list *l) -++{ -++ struct list_head *inactive = &l->lists[BPF_LRU_LIST_T_INACTIVE]; -++ struct list_head *cur, *last, *next = inactive; -++ struct bpf_lru_node *node; -++ unsigned int i = 0; -++ -++ if (list_empty(inactive)) -++ return; -++ -++ last = l->next_inactive_rotation->next; -++ if (last == inactive) -++ last = last->next; -++ -++ cur = l->next_inactive_rotation; -++ while (i < lru->nr_scans) { -++ if (cur == inactive) { -++ cur = cur->prev; -++ continue; -++ } -++ -++ node = list_entry(cur, struct bpf_lru_node, list); -++ next = cur->prev; -++ if (bpf_lru_node_is_ref(node)) -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); -++ if (cur == last) -++ break; -++ cur = next; -++ i++; -++ } -++ -++ l->next_inactive_rotation = next; -++} -++ -++/* Shrink the inactive list. It starts from the tail of the -++ * inactive list and only move the nodes without the ref bit -++ * set to the designated free list. -++ */ -++static unsigned int -++__bpf_lru_list_shrink_inactive(struct bpf_lru *lru, -++ struct bpf_lru_list *l, -++ unsigned int tgt_nshrink, -++ struct list_head *free_list, -++ enum bpf_lru_list_type tgt_free_type) -++{ -++ struct list_head *inactive = &l->lists[BPF_LRU_LIST_T_INACTIVE]; -++ struct bpf_lru_node *node, *tmp_node; -++ unsigned int nshrinked = 0; -++ unsigned int i = 0; -++ -++ list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) { -++ if (bpf_lru_node_is_ref(node)) { -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE); -++ } else if (lru->del_from_htab(lru->del_arg, node)) { -++ __bpf_lru_node_move_to_free(l, node, free_list, -++ tgt_free_type); -++ if (++nshrinked == tgt_nshrink) -++ break; -++ } -++ -++ if (++i == lru->nr_scans) -++ break; -++ } -++ -++ return nshrinked; -++} -++ -++/* 1. Rotate the active list (if needed) -++ * 2. Always rotate the inactive list -++ */ -++static void __bpf_lru_list_rotate(struct bpf_lru *lru, struct bpf_lru_list *l) -++{ -++ if (bpf_lru_list_inactive_low(l)) -++ __bpf_lru_list_rotate_active(lru, l); -++ -++ __bpf_lru_list_rotate_inactive(lru, l); -++} -++ -++/* Calls __bpf_lru_list_shrink_inactive() to shrink some -++ * ref-bit-cleared nodes and move them to the designated -++ * free list. -++ * -++ * If it cannot get a free node after calling -++ * __bpf_lru_list_shrink_inactive(). It will just remove -++ * one node from either inactive or active list without -++ * honoring the ref-bit. It prefers inactive list to active -++ * list in this situation. -++ */ -++static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru, -++ struct bpf_lru_list *l, -++ unsigned int tgt_nshrink, -++ struct list_head *free_list, -++ enum bpf_lru_list_type tgt_free_type) -++ -++{ -++ struct bpf_lru_node *node, *tmp_node; -++ struct list_head *force_shrink_list; -++ unsigned int nshrinked; -++ -++ nshrinked = __bpf_lru_list_shrink_inactive(lru, l, tgt_nshrink, -++ free_list, tgt_free_type); -++ if (nshrinked) -++ return nshrinked; -++ -++ /* Do a force shrink by ignoring the reference bit */ -++ if (!list_empty(&l->lists[BPF_LRU_LIST_T_INACTIVE])) -++ force_shrink_list = &l->lists[BPF_LRU_LIST_T_INACTIVE]; -++ else -++ force_shrink_list = &l->lists[BPF_LRU_LIST_T_ACTIVE]; -++ -++ list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list, -++ list) { -++ if (lru->del_from_htab(lru->del_arg, node)) { -++ __bpf_lru_node_move_to_free(l, node, free_list, -++ tgt_free_type); -++ return 1; -++ } -++ } -++ -++ return 0; -++} -++ -++/* Flush the nodes from the local pending list to the LRU list */ -++static void __local_list_flush(struct bpf_lru_list *l, -++ struct bpf_lru_locallist *loc_l) -++{ -++ struct bpf_lru_node *node, *tmp_node; -++ -++ list_for_each_entry_safe_reverse(node, tmp_node, -++ local_pending_list(loc_l), list) { -++ if (bpf_lru_node_is_ref(node)) -++ __bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE); -++ else -++ __bpf_lru_node_move_in(l, node, -++ BPF_LRU_LIST_T_INACTIVE); -++ } -++} -++ -++static void bpf_lru_list_push_free(struct bpf_lru_list *l, -++ struct bpf_lru_node *node) -++{ -++ unsigned long flags; -++ -++ if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type))) -++ return; -++ -++ raw_spin_lock_irqsave(&l->lock, flags); -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); -++ raw_spin_unlock_irqrestore(&l->lock, flags); -++} -++ -++static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru, -++ struct bpf_lru_locallist *loc_l) -++{ -++ struct bpf_lru_list *l = &lru->common_lru.lru_list; -++ struct bpf_lru_node *node, *tmp_node; -++ unsigned int nfree = 0; -++ -++ raw_spin_lock(&l->lock); -++ -++ __local_list_flush(l, loc_l); -++ -++ __bpf_lru_list_rotate(lru, l); -++ -++ list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE], -++ list) { -++ __bpf_lru_node_move_to_free(l, node, local_free_list(loc_l), -++ BPF_LRU_LOCAL_LIST_T_FREE); -++ if (++nfree == LOCAL_FREE_TARGET) -++ break; -++ } -++ -++ if (nfree < LOCAL_FREE_TARGET) -++ __bpf_lru_list_shrink(lru, l, LOCAL_FREE_TARGET - nfree, -++ local_free_list(loc_l), -++ BPF_LRU_LOCAL_LIST_T_FREE); -++ -++ raw_spin_unlock(&l->lock); -++} -++ -++static void __local_list_add_pending(struct bpf_lru *lru, -++ struct bpf_lru_locallist *loc_l, -++ int cpu, -++ struct bpf_lru_node *node, -++ u32 hash) -++{ -++ *(u32 *)((void *)node + lru->hash_offset) = hash; -++ node->cpu = cpu; -++ node->type = BPF_LRU_LOCAL_LIST_T_PENDING; -++ node->ref = 0; -++ list_add(&node->list, local_pending_list(loc_l)); -++} -++ -++static struct bpf_lru_node * -++__local_list_pop_free(struct bpf_lru_locallist *loc_l) -++{ -++ struct bpf_lru_node *node; -++ -++ node = list_first_entry_or_null(local_free_list(loc_l), -++ struct bpf_lru_node, -++ list); -++ if (node) -++ list_del(&node->list); -++ -++ return node; -++} -++ -++static struct bpf_lru_node * -++__local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l) -++{ -++ struct bpf_lru_node *node; -++ bool force = false; -++ -++ignore_ref: -++ /* Get from the tail (i.e. older element) of the pending list. */ -++ list_for_each_entry_reverse(node, local_pending_list(loc_l), -++ list) { -++ if ((!bpf_lru_node_is_ref(node) || force) && -++ lru->del_from_htab(lru->del_arg, node)) { -++ list_del(&node->list); -++ return node; -++ } -++ } -++ -++ if (!force) { -++ force = true; -++ goto ignore_ref; -++ } -++ -++ return NULL; -++} -++ -++static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru, -++ u32 hash) -++{ -++ struct list_head *free_list; -++ struct bpf_lru_node *node = NULL; -++ struct bpf_lru_list *l; -++ unsigned long flags; -++ int cpu = raw_smp_processor_id(); -++ -++ l = per_cpu_ptr(lru->percpu_lru, cpu); -++ -++ raw_spin_lock_irqsave(&l->lock, flags); -++ -++ __bpf_lru_list_rotate(lru, l); -++ -++ free_list = &l->lists[BPF_LRU_LIST_T_FREE]; -++ if (list_empty(free_list)) -++ __bpf_lru_list_shrink(lru, l, PERCPU_FREE_TARGET, free_list, -++ BPF_LRU_LIST_T_FREE); -++ -++ if (!list_empty(free_list)) { -++ node = list_first_entry(free_list, struct bpf_lru_node, list); -++ *(u32 *)((void *)node + lru->hash_offset) = hash; -++ node->ref = 0; -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE); -++ } -++ -++ raw_spin_unlock_irqrestore(&l->lock, flags); -++ -++ return node; -++} -++ -++static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru, -++ u32 hash) -++{ -++ struct bpf_lru_locallist *loc_l, *steal_loc_l; -++ struct bpf_common_lru *clru = &lru->common_lru; -++ struct bpf_lru_node *node; -++ int steal, first_steal; -++ unsigned long flags; -++ int cpu = raw_smp_processor_id(); -++ -++ loc_l = per_cpu_ptr(clru->local_list, cpu); -++ -++ raw_spin_lock_irqsave(&loc_l->lock, flags); -++ -++ node = __local_list_pop_free(loc_l); -++ if (!node) { -++ bpf_lru_list_pop_free_to_local(lru, loc_l); -++ node = __local_list_pop_free(loc_l); -++ } -++ -++ if (node) -++ __local_list_add_pending(lru, loc_l, cpu, node, hash); -++ -++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); -++ -++ if (node) -++ return node; -++ -++ /* No free nodes found from the local free list and -++ * the global LRU list. -++ * -++ * Steal from the local free/pending list of the -++ * current CPU and remote CPU in RR. It starts -++ * with the loc_l->next_steal CPU. -++ */ -++ -++ first_steal = loc_l->next_steal; -++ steal = first_steal; -++ do { -++ steal_loc_l = per_cpu_ptr(clru->local_list, steal); -++ -++ raw_spin_lock_irqsave(&steal_loc_l->lock, flags); -++ -++ node = __local_list_pop_free(steal_loc_l); -++ if (!node) -++ node = __local_list_pop_pending(lru, steal_loc_l); -++ -++ raw_spin_unlock_irqrestore(&steal_loc_l->lock, flags); -++ -++ steal = get_next_cpu(steal); -++ } while (!node && steal != first_steal); -++ -++ loc_l->next_steal = steal; -++ -++ if (node) { -++ raw_spin_lock_irqsave(&loc_l->lock, flags); -++ __local_list_add_pending(lru, loc_l, cpu, node, hash); -++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); -++ } -++ -++ return node; -++} -++ -++struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) -++{ -++ if (lru->percpu) -++ return bpf_percpu_lru_pop_free(lru, hash); -++ else -++ return bpf_common_lru_pop_free(lru, hash); -++} -++ -++static void bpf_common_lru_push_free(struct bpf_lru *lru, -++ struct bpf_lru_node *node) -++{ -++ u8 node_type = READ_ONCE(node->type); -++ unsigned long flags; -++ -++ if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) || -++ WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE)) -++ return; -++ -++ if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) { -++ struct bpf_lru_locallist *loc_l; -++ -++ loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu); -++ -++ raw_spin_lock_irqsave(&loc_l->lock, flags); -++ -++ if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) { -++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); -++ goto check_lru_list; -++ } -++ -++ node->type = BPF_LRU_LOCAL_LIST_T_FREE; -++ node->ref = 0; -++ list_move(&node->list, local_free_list(loc_l)); -++ -++ raw_spin_unlock_irqrestore(&loc_l->lock, flags); -++ return; -++ } -++ -++check_lru_list: -++ bpf_lru_list_push_free(&lru->common_lru.lru_list, node); -++} -++ -++static void bpf_percpu_lru_push_free(struct bpf_lru *lru, -++ struct bpf_lru_node *node) -++{ -++ struct bpf_lru_list *l; -++ unsigned long flags; -++ -++ l = per_cpu_ptr(lru->percpu_lru, node->cpu); -++ -++ raw_spin_lock_irqsave(&l->lock, flags); -++ -++ __bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE); -++ -++ raw_spin_unlock_irqrestore(&l->lock, flags); -++} -++ -++void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node) -++{ -++ if (lru->percpu) -++ bpf_percpu_lru_push_free(lru, node); -++ else -++ bpf_common_lru_push_free(lru, node); -++} -++ -++static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf, -++ u32 node_offset, u32 elem_size, -++ u32 nr_elems) -++{ -++ struct bpf_lru_list *l = &lru->common_lru.lru_list; -++ u32 i; -++ -++ for (i = 0; i < nr_elems; i++) { -++ struct bpf_lru_node *node; -++ -++ node = (struct bpf_lru_node *)(buf + node_offset); -++ node->type = BPF_LRU_LIST_T_FREE; -++ node->ref = 0; -++ list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); -++ buf += elem_size; -++ } -++} -++ -++static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf, -++ u32 node_offset, u32 elem_size, -++ u32 nr_elems) -++{ -++ u32 i, pcpu_entries; -++ int cpu; -++ struct bpf_lru_list *l; -++ -++ pcpu_entries = nr_elems / num_possible_cpus(); -++ -++ i = 0; -++ -++ for_each_possible_cpu(cpu) { -++ struct bpf_lru_node *node; -++ -++ l = per_cpu_ptr(lru->percpu_lru, cpu); -++again: -++ node = (struct bpf_lru_node *)(buf + node_offset); -++ node->cpu = cpu; -++ node->type = BPF_LRU_LIST_T_FREE; -++ node->ref = 0; -++ list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]); -++ i++; -++ buf += elem_size; -++ if (i == nr_elems) -++ break; -++ if (i % pcpu_entries) -++ goto again; -++ } -++} -++ -++void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, -++ u32 elem_size, u32 nr_elems) -++{ -++ if (lru->percpu) -++ bpf_percpu_lru_populate(lru, buf, node_offset, elem_size, -++ nr_elems); -++ else -++ bpf_common_lru_populate(lru, buf, node_offset, elem_size, -++ nr_elems); -++} -++ -++static void bpf_lru_locallist_init(struct bpf_lru_locallist *loc_l, int cpu) -++{ -++ int i; -++ -++ for (i = 0; i < NR_BPF_LRU_LOCAL_LIST_T; i++) -++ INIT_LIST_HEAD(&loc_l->lists[i]); -++ -++ loc_l->next_steal = cpu; -++ -++ raw_spin_lock_init(&loc_l->lock); -++} -++ -++static void bpf_lru_list_init(struct bpf_lru_list *l) -++{ -++ int i; -++ -++ for (i = 0; i < NR_BPF_LRU_LIST_T; i++) -++ INIT_LIST_HEAD(&l->lists[i]); -++ -++ for (i = 0; i < NR_BPF_LRU_LIST_COUNT; i++) -++ l->counts[i] = 0; -++ -++ l->next_inactive_rotation = &l->lists[BPF_LRU_LIST_T_INACTIVE]; -++ -++ raw_spin_lock_init(&l->lock); -++} -++ -++int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, -++ del_from_htab_func del_from_htab, void *del_arg) -++{ -++ int cpu; -++ -++ if (percpu) { -++ lru->percpu_lru = alloc_percpu(struct bpf_lru_list); -++ if (!lru->percpu_lru) -++ return -ENOMEM; -++ -++ for_each_possible_cpu(cpu) { -++ struct bpf_lru_list *l; -++ -++ l = per_cpu_ptr(lru->percpu_lru, cpu); -++ bpf_lru_list_init(l); -++ } -++ lru->nr_scans = PERCPU_NR_SCANS; -++ } else { -++ struct bpf_common_lru *clru = &lru->common_lru; -++ -++ clru->local_list = alloc_percpu(struct bpf_lru_locallist); -++ if (!clru->local_list) -++ return -ENOMEM; -++ -++ for_each_possible_cpu(cpu) { -++ struct bpf_lru_locallist *loc_l; -++ -++ loc_l = per_cpu_ptr(clru->local_list, cpu); -++ bpf_lru_locallist_init(loc_l, cpu); -++ } -++ -++ bpf_lru_list_init(&clru->lru_list); -++ lru->nr_scans = LOCAL_NR_SCANS; -++ } -++ -++ lru->percpu = percpu; -++ lru->del_from_htab = del_from_htab; -++ lru->del_arg = del_arg; -++ lru->hash_offset = hash_offset; -++ -++ return 0; -++} -++ -++void bpf_lru_destroy(struct bpf_lru *lru) -++{ -++ if (lru->percpu) -++ free_percpu(lru->percpu_lru); -++ else -++ free_percpu(lru->common_lru.local_list); -++} -+--- /dev/null -++++ b/kernel/bpf/bpf_lru_list.h -+@@ -0,0 +1,82 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* Copyright (c) 2016 Facebook -++ */ -++#ifndef __BPF_LRU_LIST_H_ -++#define __BPF_LRU_LIST_H_ -++ -++#include -++#include -++ -++#define NR_BPF_LRU_LIST_T (3) -++#define NR_BPF_LRU_LIST_COUNT (2) -++#define NR_BPF_LRU_LOCAL_LIST_T (2) -++#define BPF_LOCAL_LIST_T_OFFSET NR_BPF_LRU_LIST_T -++ -++enum bpf_lru_list_type { -++ BPF_LRU_LIST_T_ACTIVE, -++ BPF_LRU_LIST_T_INACTIVE, -++ BPF_LRU_LIST_T_FREE, -++ BPF_LRU_LOCAL_LIST_T_FREE, -++ BPF_LRU_LOCAL_LIST_T_PENDING, -++}; -++ -++struct bpf_lru_node { -++ struct list_head list; -++ u16 cpu; -++ u8 type; -++ u8 ref; -++}; -++ -++struct bpf_lru_list { -++ struct list_head lists[NR_BPF_LRU_LIST_T]; -++ unsigned int counts[NR_BPF_LRU_LIST_COUNT]; -++ /* The next inacitve list rotation starts from here */ -++ struct list_head *next_inactive_rotation; -++ -++ raw_spinlock_t lock ____cacheline_aligned_in_smp; -++}; -++ -++struct bpf_lru_locallist { -++ struct list_head lists[NR_BPF_LRU_LOCAL_LIST_T]; -++ u16 next_steal; -++ raw_spinlock_t lock; -++}; -++ -++struct bpf_common_lru { -++ struct bpf_lru_list lru_list; -++ struct bpf_lru_locallist __percpu *local_list; -++}; -++ -++typedef bool (*del_from_htab_func)(void *arg, struct bpf_lru_node *node); -++ -++struct bpf_lru { -++ union { -++ struct bpf_common_lru common_lru; -++ struct bpf_lru_list __percpu *percpu_lru; -++ }; -++ del_from_htab_func del_from_htab; -++ void *del_arg; -++ unsigned int hash_offset; -++ unsigned int nr_scans; -++ bool percpu; -++}; -++ -++static inline void bpf_lru_node_set_ref(struct bpf_lru_node *node) -++{ -++ /* ref is an approximation on access frequency. It does not -++ * have to be very accurate. Hence, no protection is used. -++ */ -++ if (!node->ref) -++ node->ref = 1; -++} -++ -++int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset, -++ del_from_htab_func del_from_htab, void *delete_arg); -++void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset, -++ u32 elem_size, u32 nr_elems); -++void bpf_lru_destroy(struct bpf_lru *lru); -++struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash); -++void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node); -++void bpf_lru_promote(struct bpf_lru *lru, struct bpf_lru_node *node); -++ -++#endif -+--- /dev/null -++++ b/kernel/bpf/btf.c -+@@ -0,0 +1,3514 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++/* Copyright (c) 2018 Facebook */ -++ -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++/* BTF (BPF Type Format) is the meta data format which describes -++ * the data types of BPF program/map. Hence, it basically focus -++ * on the C programming language which the modern BPF is primary -++ * using. -++ * -++ * ELF Section: -++ * ~~~~~~~~~~~ -++ * The BTF data is stored under the ".BTF" ELF section -++ * -++ * struct btf_type: -++ * ~~~~~~~~~~~~~~~ -++ * Each 'struct btf_type' object describes a C data type. -++ * Depending on the type it is describing, a 'struct btf_type' -++ * object may be followed by more data. F.e. -++ * To describe an array, 'struct btf_type' is followed by -++ * 'struct btf_array'. -++ * -++ * 'struct btf_type' and any extra data following it are -++ * 4 bytes aligned. -++ * -++ * Type section: -++ * ~~~~~~~~~~~~~ -++ * The BTF type section contains a list of 'struct btf_type' objects. -++ * Each one describes a C type. Recall from the above section -++ * that a 'struct btf_type' object could be immediately followed by extra -++ * data in order to desribe some particular C types. -++ * -++ * type_id: -++ * ~~~~~~~ -++ * Each btf_type object is identified by a type_id. The type_id -++ * is implicitly implied by the location of the btf_type object in -++ * the BTF type section. The first one has type_id 1. The second -++ * one has type_id 2...etc. Hence, an earlier btf_type has -++ * a smaller type_id. -++ * -++ * A btf_type object may refer to another btf_type object by using -++ * type_id (i.e. the "type" in the "struct btf_type"). -++ * -++ * NOTE that we cannot assume any reference-order. -++ * A btf_type object can refer to an earlier btf_type object -++ * but it can also refer to a later btf_type object. -++ * -++ * For example, to describe "const void *". A btf_type -++ * object describing "const" may refer to another btf_type -++ * object describing "void *". This type-reference is done -++ * by specifying type_id: -++ * -++ * [1] CONST (anon) type_id=2 -++ * [2] PTR (anon) type_id=0 -++ * -++ * The above is the btf_verifier debug log: -++ * - Each line started with "[?]" is a btf_type object -++ * - [?] is the type_id of the btf_type object. -++ * - CONST/PTR is the BTF_KIND_XXX -++ * - "(anon)" is the name of the type. It just -++ * happens that CONST and PTR has no name. -++ * - type_id=XXX is the 'u32 type' in btf_type -++ * -++ * NOTE: "void" has type_id 0 -++ * -++ * String section: -++ * ~~~~~~~~~~~~~~ -++ * The BTF string section contains the names used by the type section. -++ * Each string is referred by an "offset" from the beginning of the -++ * string section. -++ * -++ * Each string is '\0' terminated. -++ * -++ * The first character in the string section must be '\0' -++ * which is used to mean 'anonymous'. Some btf_type may not -++ * have a name. -++ */ -++ -++/* BTF verification: -++ * -++ * To verify BTF data, two passes are needed. -++ * -++ * Pass #1 -++ * ~~~~~~~ -++ * The first pass is to collect all btf_type objects to -++ * an array: "btf->types". -++ * -++ * Depending on the C type that a btf_type is describing, -++ * a btf_type may be followed by extra data. We don't know -++ * how many btf_type is there, and more importantly we don't -++ * know where each btf_type is located in the type section. -++ * -++ * Without knowing the location of each type_id, most verifications -++ * cannot be done. e.g. an earlier btf_type may refer to a later -++ * btf_type (recall the "const void *" above), so we cannot -++ * check this type-reference in the first pass. -++ * -++ * In the first pass, it still does some verifications (e.g. -++ * checking the name is a valid offset to the string section). -++ * -++ * Pass #2 -++ * ~~~~~~~ -++ * The main focus is to resolve a btf_type that is referring -++ * to another type. -++ * -++ * We have to ensure the referring type: -++ * 1) does exist in the BTF (i.e. in btf->types[]) -++ * 2) does not cause a loop: -++ * struct A { -++ * struct B b; -++ * }; -++ * -++ * struct B { -++ * struct A a; -++ * }; -++ * -++ * btf_type_needs_resolve() decides if a btf_type needs -++ * to be resolved. -++ * -++ * The needs_resolve type implements the "resolve()" ops which -++ * essentially does a DFS and detects backedge. -++ * -++ * During resolve (or DFS), different C types have different -++ * "RESOLVED" conditions. -++ * -++ * When resolving a BTF_KIND_STRUCT, we need to resolve all its -++ * members because a member is always referring to another -++ * type. A struct's member can be treated as "RESOLVED" if -++ * it is referring to a BTF_KIND_PTR. Otherwise, the -++ * following valid C struct would be rejected: -++ * -++ * struct A { -++ * int m; -++ * struct A *a; -++ * }; -++ * -++ * When resolving a BTF_KIND_PTR, it needs to keep resolving if -++ * it is referring to another BTF_KIND_PTR. Otherwise, we cannot -++ * detect a pointer loop, e.g.: -++ * BTF_KIND_CONST -> BTF_KIND_PTR -> BTF_KIND_CONST -> BTF_KIND_PTR + -++ * ^ | -++ * +-----------------------------------------+ -++ * -++ */ -++ -++#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2) -++#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1) -++#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK) -++#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3) -++#define BITS_ROUNDUP_BYTES(bits) \ -++ (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits)) -++ -++#define BTF_INFO_MASK 0x8f00ffff -++#define BTF_INT_MASK 0x0fffffff -++#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE) -++#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET) -++ -++/* 16MB for 64k structs and each has 16 members and -++ * a few MB spaces for the string section. -++ * The hard limit is S32_MAX. -++ */ -++#define BTF_MAX_SIZE (16 * 1024 * 1024) -++ -++#define for_each_member(i, struct_type, member) \ -++ for (i = 0, member = btf_type_member(struct_type); \ -++ i < btf_type_vlen(struct_type); \ -++ i++, member++) -++ -++#define for_each_member_from(i, from, struct_type, member) \ -++ for (i = from, member = btf_type_member(struct_type) + from; \ -++ i < btf_type_vlen(struct_type); \ -++ i++, member++) -++ -++#define for_each_vsi(i, struct_type, member) \ -++ for (i = 0, member = btf_type_var_secinfo(struct_type); \ -++ i < btf_type_vlen(struct_type); \ -++ i++, member++) -++ -++#define for_each_vsi_from(i, from, struct_type, member) \ -++ for (i = from, member = btf_type_var_secinfo(struct_type) + from; \ -++ i < btf_type_vlen(struct_type); \ -++ i++, member++) -++ -++DEFINE_IDR(btf_idr); -++DEFINE_SPINLOCK(btf_idr_lock); -++ -++struct btf { -++ void *data; -++ struct btf_type **types; -++ u32 *resolved_ids; -++ u32 *resolved_sizes; -++ const char *strings; -++ void *nohdr_data; -++ struct btf_header hdr; -++ u32 nr_types; -++ u32 types_size; -++ u32 data_size; -++ refcount_t refcnt; -++ u32 id; -++ struct rcu_head rcu; -++}; -++ -++enum verifier_phase { -++ CHECK_META, -++ CHECK_TYPE, -++}; -++ -++struct resolve_vertex { -++ const struct btf_type *t; -++ u32 type_id; -++ u16 next_member; -++}; -++ -++enum visit_state { -++ NOT_VISITED, -++ VISITED, -++ RESOLVED, -++}; -++ -++enum resolve_mode { -++ RESOLVE_TBD, /* To Be Determined */ -++ RESOLVE_PTR, /* Resolving for Pointer */ -++ RESOLVE_STRUCT_OR_ARRAY, /* Resolving for struct/union -++ * or array -++ */ -++}; -++ -++#define MAX_RESOLVE_DEPTH 32 -++ -++struct btf_sec_info { -++ u32 off; -++ u32 len; -++}; -++ -++struct btf_verifier_env { -++ struct btf *btf; -++ u8 *visit_states; -++ struct resolve_vertex stack[MAX_RESOLVE_DEPTH]; -++ struct bpf_verifier_log log; -++ u32 log_type_id; -++ u32 top_stack; -++ enum verifier_phase phase; -++ enum resolve_mode resolve_mode; -++}; -++ -++static const char * const btf_kind_str[NR_BTF_KINDS] = { -++ [BTF_KIND_UNKN] = "UNKNOWN", -++ [BTF_KIND_INT] = "INT", -++ [BTF_KIND_PTR] = "PTR", -++ [BTF_KIND_ARRAY] = "ARRAY", -++ [BTF_KIND_STRUCT] = "STRUCT", -++ [BTF_KIND_UNION] = "UNION", -++ [BTF_KIND_ENUM] = "ENUM", -++ [BTF_KIND_FWD] = "FWD", -++ [BTF_KIND_TYPEDEF] = "TYPEDEF", -++ [BTF_KIND_VOLATILE] = "VOLATILE", -++ [BTF_KIND_CONST] = "CONST", -++ [BTF_KIND_RESTRICT] = "RESTRICT", -++ [BTF_KIND_FUNC] = "FUNC", -++ [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO", -++ [BTF_KIND_VAR] = "VAR", -++ [BTF_KIND_DATASEC] = "DATASEC", -++}; -++ -++struct btf_kind_operations { -++ s32 (*check_meta)(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left); -++ int (*resolve)(struct btf_verifier_env *env, -++ const struct resolve_vertex *v); -++ int (*check_member)(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type); -++ int (*check_kflag_member)(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type); -++ void (*log_details)(struct btf_verifier_env *env, -++ const struct btf_type *t); -++ void (*seq_show)(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offsets, -++ struct seq_file *m); -++}; -++ -++static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS]; -++static struct btf_type btf_void; -++ -++static int btf_resolve(struct btf_verifier_env *env, -++ const struct btf_type *t, u32 type_id); -++ -++static bool btf_type_is_modifier(const struct btf_type *t) -++{ -++ /* Some of them is not strictly a C modifier -++ * but they are grouped into the same bucket -++ * for BTF concern: -++ * A type (t) that refers to another -++ * type through t->type AND its size cannot -++ * be determined without following the t->type. -++ * -++ * ptr does not fall into this bucket -++ * because its size is always sizeof(void *). -++ */ -++ switch (BTF_INFO_KIND(t->info)) { -++ case BTF_KIND_TYPEDEF: -++ case BTF_KIND_VOLATILE: -++ case BTF_KIND_CONST: -++ case BTF_KIND_RESTRICT: -++ return true; -++ } -++ -++ return false; -++} -++ -++bool btf_type_is_void(const struct btf_type *t) -++{ -++ return t == &btf_void; -++} -++ -++static bool btf_type_is_fwd(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_FWD; -++} -++ -++static bool btf_type_is_func(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC; -++} -++ -++static bool btf_type_is_func_proto(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO; -++} -++ -++static bool btf_type_nosize(const struct btf_type *t) -++{ -++ return btf_type_is_void(t) || btf_type_is_fwd(t) || -++ btf_type_is_func(t) || btf_type_is_func_proto(t); -++} -++ -++static bool btf_type_nosize_or_null(const struct btf_type *t) -++{ -++ return !t || btf_type_nosize(t); -++} -++ -++/* union is only a special case of struct: -++ * all its offsetof(member) == 0 -++ */ -++static bool btf_type_is_struct(const struct btf_type *t) -++{ -++ u8 kind = BTF_INFO_KIND(t->info); -++ -++ return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; -++} -++ -++static bool __btf_type_is_struct(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT; -++} -++ -++static bool btf_type_is_array(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; -++} -++ -++static bool btf_type_is_ptr(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; -++} -++ -++static bool btf_type_is_int(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_INT; -++} -++ -++static bool btf_type_is_var(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_VAR; -++} -++ -++static bool btf_type_is_datasec(const struct btf_type *t) -++{ -++ return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC; -++} -++ -++/* Types that act only as a source, not sink or intermediate -++ * type when resolving. -++ */ -++static bool btf_type_is_resolve_source_only(const struct btf_type *t) -++{ -++ return btf_type_is_var(t) || -++ btf_type_is_datasec(t); -++} -++ -++/* What types need to be resolved? -++ * -++ * btf_type_is_modifier() is an obvious one. -++ * -++ * btf_type_is_struct() because its member refers to -++ * another type (through member->type). -++ * -++ * btf_type_is_var() because the variable refers to -++ * another type. btf_type_is_datasec() holds multiple -++ * btf_type_is_var() types that need resolving. -++ * -++ * btf_type_is_array() because its element (array->type) -++ * refers to another type. Array can be thought of a -++ * special case of struct while array just has the same -++ * member-type repeated by array->nelems of times. -++ */ -++static bool btf_type_needs_resolve(const struct btf_type *t) -++{ -++ return btf_type_is_modifier(t) || -++ btf_type_is_ptr(t) || -++ btf_type_is_struct(t) || -++ btf_type_is_array(t) || -++ btf_type_is_var(t) || -++ btf_type_is_datasec(t); -++} -++ -++/* t->size can be used */ -++static bool btf_type_has_size(const struct btf_type *t) -++{ -++ switch (BTF_INFO_KIND(t->info)) { -++ case BTF_KIND_INT: -++ case BTF_KIND_STRUCT: -++ case BTF_KIND_UNION: -++ case BTF_KIND_ENUM: -++ case BTF_KIND_DATASEC: -++ return true; -++ } -++ -++ return false; -++} -++ -++static const char *btf_int_encoding_str(u8 encoding) -++{ -++ if (encoding == 0) -++ return "(none)"; -++ else if (encoding == BTF_INT_SIGNED) -++ return "SIGNED"; -++ else if (encoding == BTF_INT_CHAR) -++ return "CHAR"; -++ else if (encoding == BTF_INT_BOOL) -++ return "BOOL"; -++ else -++ return "UNKN"; -++} -++ -++static u16 btf_type_vlen(const struct btf_type *t) -++{ -++ return BTF_INFO_VLEN(t->info); -++} -++ -++static bool btf_type_kflag(const struct btf_type *t) -++{ -++ return BTF_INFO_KFLAG(t->info); -++} -++ -++static u32 btf_member_bit_offset(const struct btf_type *struct_type, -++ const struct btf_member *member) -++{ -++ return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset) -++ : member->offset; -++} -++ -++static u32 btf_member_bitfield_size(const struct btf_type *struct_type, -++ const struct btf_member *member) -++{ -++ return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset) -++ : 0; -++} -++ -++static u32 btf_type_int(const struct btf_type *t) -++{ -++ return *(u32 *)(t + 1); -++} -++ -++static const struct btf_array *btf_type_array(const struct btf_type *t) -++{ -++ return (const struct btf_array *)(t + 1); -++} -++ -++static const struct btf_member *btf_type_member(const struct btf_type *t) -++{ -++ return (const struct btf_member *)(t + 1); -++} -++ -++static const struct btf_enum *btf_type_enum(const struct btf_type *t) -++{ -++ return (const struct btf_enum *)(t + 1); -++} -++ -++static const struct btf_var *btf_type_var(const struct btf_type *t) -++{ -++ return (const struct btf_var *)(t + 1); -++} -++ -++static const struct btf_var_secinfo *btf_type_var_secinfo(const struct btf_type *t) -++{ -++ return (const struct btf_var_secinfo *)(t + 1); -++} -++ -++static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) -++{ -++ return kind_ops[BTF_INFO_KIND(t->info)]; -++} -++ -++static bool btf_name_offset_valid(const struct btf *btf, u32 offset) -++{ -++ return BTF_STR_OFFSET_VALID(offset) && -++ offset < btf->hdr.str_len; -++} -++ -++static bool __btf_name_char_ok(char c, bool first, bool dot_ok) -++{ -++ if ((first ? !isalpha(c) : -++ !isalnum(c)) && -++ c != '_' && -++ ((c == '.' && !dot_ok) || -++ c != '.')) -++ return false; -++ return true; -++} -++ -++static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok) -++{ -++ /* offset must be valid */ -++ const char *src = &btf->strings[offset]; -++ const char *src_limit; -++ -++ if (!__btf_name_char_ok(*src, true, dot_ok)) -++ return false; -++ -++ /* set a limit on identifier length */ -++ src_limit = src + KSYM_NAME_LEN; -++ src++; -++ while (*src && src < src_limit) { -++ if (!__btf_name_char_ok(*src, false, dot_ok)) -++ return false; -++ src++; -++ } -++ -++ return !*src; -++} -++ -++/* Only C-style identifier is permitted. This can be relaxed if -++ * necessary. -++ */ -++static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) -++{ -++ return __btf_name_valid(btf, offset, false); -++} -++ -++static bool btf_name_valid_section(const struct btf *btf, u32 offset) -++{ -++ return __btf_name_valid(btf, offset, true); -++} -++ -++static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) -++{ -++ if (!offset) -++ return "(anon)"; -++ else if (offset < btf->hdr.str_len) -++ return &btf->strings[offset]; -++ else -++ return "(invalid-name-offset)"; -++} -++ -++const char *btf_name_by_offset(const struct btf *btf, u32 offset) -++{ -++ if (offset < btf->hdr.str_len) -++ return &btf->strings[offset]; -++ -++ return NULL; -++} -++ -++const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) -++{ -++ if (type_id > btf->nr_types) -++ return NULL; -++ -++ return btf->types[type_id]; -++} -++ -++/* -++ * Regular int is not a bit field and it must be either -++ * u8/u16/u32/u64 or __int128. -++ */ -++static bool btf_type_int_is_regular(const struct btf_type *t) -++{ -++ u8 nr_bits, nr_bytes; -++ u32 int_data; -++ -++ int_data = btf_type_int(t); -++ nr_bits = BTF_INT_BITS(int_data); -++ nr_bytes = BITS_ROUNDUP_BYTES(nr_bits); -++ if (BITS_PER_BYTE_MASKED(nr_bits) || -++ BTF_INT_OFFSET(int_data) || -++ (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) && -++ nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) && -++ nr_bytes != (2 * sizeof(u64)))) { -++ return false; -++ } -++ -++ return true; -++} -++ -++/* -++ * Check that given struct member is a regular int with expected -++ * offset and size. -++ */ -++bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, -++ const struct btf_member *m, -++ u32 expected_offset, u32 expected_size) -++{ -++ const struct btf_type *t; -++ u32 id, int_data; -++ u8 nr_bits; -++ -++ id = m->type; -++ t = btf_type_id_size(btf, &id, NULL); -++ if (!t || !btf_type_is_int(t)) -++ return false; -++ -++ int_data = btf_type_int(t); -++ nr_bits = BTF_INT_BITS(int_data); -++ if (btf_type_kflag(s)) { -++ u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset); -++ u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset); -++ -++ /* if kflag set, int should be a regular int and -++ * bit offset should be at byte boundary. -++ */ -++ return !bitfield_size && -++ BITS_ROUNDUP_BYTES(bit_offset) == expected_offset && -++ BITS_ROUNDUP_BYTES(nr_bits) == expected_size; -++ } -++ -++ if (BTF_INT_OFFSET(int_data) || -++ BITS_PER_BYTE_MASKED(m->offset) || -++ BITS_ROUNDUP_BYTES(m->offset) != expected_offset || -++ BITS_PER_BYTE_MASKED(nr_bits) || -++ BITS_ROUNDUP_BYTES(nr_bits) != expected_size) -++ return false; -++ -++ return true; -++} -++ -++__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log, -++ const char *fmt, ...) -++{ -++ va_list args; -++ -++ va_start(args, fmt); -++ bpf_verifier_vlog(log, fmt, args); -++ va_end(args); -++} -++ -++__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env, -++ const char *fmt, ...) -++{ -++ struct bpf_verifier_log *log = &env->log; -++ va_list args; -++ -++ if (!bpf_verifier_log_needed(log)) -++ return; -++ -++ va_start(args, fmt); -++ bpf_verifier_vlog(log, fmt, args); -++ va_end(args); -++} -++ -++__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ bool log_details, -++ const char *fmt, ...) -++{ -++ struct bpf_verifier_log *log = &env->log; -++ u8 kind = BTF_INFO_KIND(t->info); -++ struct btf *btf = env->btf; -++ va_list args; -++ -++ if (!bpf_verifier_log_needed(log)) -++ return; -++ -++ __btf_verifier_log(log, "[%u] %s %s%s", -++ env->log_type_id, -++ btf_kind_str[kind], -++ __btf_name_by_offset(btf, t->name_off), -++ log_details ? " " : ""); -++ -++ if (log_details) -++ btf_type_ops(t)->log_details(env, t); -++ -++ if (fmt && *fmt) { -++ __btf_verifier_log(log, " "); -++ va_start(args, fmt); -++ bpf_verifier_vlog(log, fmt, args); -++ va_end(args); -++ } -++ -++ __btf_verifier_log(log, "\n"); -++} -++ -++#define btf_verifier_log_type(env, t, ...) \ -++ __btf_verifier_log_type((env), (t), true, __VA_ARGS__) -++#define btf_verifier_log_basic(env, t, ...) \ -++ __btf_verifier_log_type((env), (t), false, __VA_ARGS__) -++ -++__printf(4, 5) -++static void btf_verifier_log_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const char *fmt, ...) -++{ -++ struct bpf_verifier_log *log = &env->log; -++ struct btf *btf = env->btf; -++ va_list args; -++ -++ if (!bpf_verifier_log_needed(log)) -++ return; -++ -++ /* The CHECK_META phase already did a btf dump. -++ * -++ * If member is logged again, it must hit an error in -++ * parsing this member. It is useful to print out which -++ * struct this member belongs to. -++ */ -++ if (env->phase != CHECK_META) -++ btf_verifier_log_type(env, struct_type, NULL); -++ -++ if (btf_type_kflag(struct_type)) -++ __btf_verifier_log(log, -++ "\t%s type_id=%u bitfield_size=%u bits_offset=%u", -++ __btf_name_by_offset(btf, member->name_off), -++ member->type, -++ BTF_MEMBER_BITFIELD_SIZE(member->offset), -++ BTF_MEMBER_BIT_OFFSET(member->offset)); -++ else -++ __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u", -++ __btf_name_by_offset(btf, member->name_off), -++ member->type, member->offset); -++ -++ if (fmt && *fmt) { -++ __btf_verifier_log(log, " "); -++ va_start(args, fmt); -++ bpf_verifier_vlog(log, fmt, args); -++ va_end(args); -++ } -++ -++ __btf_verifier_log(log, "\n"); -++} -++ -++__printf(4, 5) -++static void btf_verifier_log_vsi(struct btf_verifier_env *env, -++ const struct btf_type *datasec_type, -++ const struct btf_var_secinfo *vsi, -++ const char *fmt, ...) -++{ -++ struct bpf_verifier_log *log = &env->log; -++ va_list args; -++ -++ if (!bpf_verifier_log_needed(log)) -++ return; -++ if (env->phase != CHECK_META) -++ btf_verifier_log_type(env, datasec_type, NULL); -++ -++ __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u", -++ vsi->type, vsi->offset, vsi->size); -++ if (fmt && *fmt) { -++ __btf_verifier_log(log, " "); -++ va_start(args, fmt); -++ bpf_verifier_vlog(log, fmt, args); -++ va_end(args); -++ } -++ -++ __btf_verifier_log(log, "\n"); -++} -++ -++static void btf_verifier_log_hdr(struct btf_verifier_env *env, -++ u32 btf_data_size) -++{ -++ struct bpf_verifier_log *log = &env->log; -++ const struct btf *btf = env->btf; -++ const struct btf_header *hdr; -++ -++ if (!bpf_verifier_log_needed(log)) -++ return; -++ -++ hdr = &btf->hdr; -++ __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic); -++ __btf_verifier_log(log, "version: %u\n", hdr->version); -++ __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags); -++ __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len); -++ __btf_verifier_log(log, "type_off: %u\n", hdr->type_off); -++ __btf_verifier_log(log, "type_len: %u\n", hdr->type_len); -++ __btf_verifier_log(log, "str_off: %u\n", hdr->str_off); -++ __btf_verifier_log(log, "str_len: %u\n", hdr->str_len); -++ __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size); -++} -++ -++static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) -++{ -++ struct btf *btf = env->btf; -++ -++ /* < 2 because +1 for btf_void which is always in btf->types[0]. -++ * btf_void is not accounted in btf->nr_types because btf_void -++ * does not come from the BTF file. -++ */ -++ if (btf->types_size - btf->nr_types < 2) { -++ /* Expand 'types' array */ -++ -++ struct btf_type **new_types; -++ u32 expand_by, new_size; -++ -++ if (btf->types_size == BTF_MAX_TYPE) { -++ btf_verifier_log(env, "Exceeded max num of types"); -++ return -E2BIG; -++ } -++ -++ expand_by = max_t(u32, btf->types_size >> 2, 16); -++ new_size = min_t(u32, BTF_MAX_TYPE, -++ btf->types_size + expand_by); -++ -++ new_types = kcalloc(new_size, sizeof(*new_types), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!new_types) -++ return -ENOMEM; -++ -++ if (btf->nr_types == 0) -++ new_types[0] = &btf_void; -++ else -++ memcpy(new_types, btf->types, -++ sizeof(*btf->types) * (btf->nr_types + 1)); -++ -++ kvfree(btf->types); -++ btf->types = new_types; -++ btf->types_size = new_size; -++ } -++ -++ btf->types[++(btf->nr_types)] = t; -++ -++ return 0; -++} -++ -++static int btf_alloc_id(struct btf *btf) -++{ -++ int id; -++ -++ idr_preload(GFP_KERNEL); -++ spin_lock_bh(&btf_idr_lock); -++ id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); -++ if (id > 0) -++ btf->id = id; -++ spin_unlock_bh(&btf_idr_lock); -++ idr_preload_end(); -++ -++ if (WARN_ON_ONCE(!id)) -++ return -ENOSPC; -++ -++ return id > 0 ? 0 : id; -++} -++ -++static void btf_free_id(struct btf *btf) -++{ -++ unsigned long flags; -++ -++ /* -++ * In map-in-map, calling map_delete_elem() on outer -++ * map will call bpf_map_put on the inner map. -++ * It will then eventually call btf_free_id() -++ * on the inner map. Some of the map_delete_elem() -++ * implementation may have irq disabled, so -++ * we need to use the _irqsave() version instead -++ * of the _bh() version. -++ */ -++ spin_lock_irqsave(&btf_idr_lock, flags); -++ idr_remove(&btf_idr, btf->id); -++ spin_unlock_irqrestore(&btf_idr_lock, flags); -++} -++ -++static void btf_free(struct btf *btf) -++{ -++ kvfree(btf->types); -++ kvfree(btf->resolved_sizes); -++ kvfree(btf->resolved_ids); -++ kvfree(btf->data); -++ kfree(btf); -++} -++ -++static void btf_free_rcu(struct rcu_head *rcu) -++{ -++ struct btf *btf = container_of(rcu, struct btf, rcu); -++ -++ btf_free(btf); -++} -++ -++void btf_put(struct btf *btf) -++{ -++ if (btf && refcount_dec_and_test(&btf->refcnt)) { -++ btf_free_id(btf); -++ call_rcu(&btf->rcu, btf_free_rcu); -++ } -++} -++ -++static int env_resolve_init(struct btf_verifier_env *env) -++{ -++ struct btf *btf = env->btf; -++ u32 nr_types = btf->nr_types; -++ u32 *resolved_sizes = NULL; -++ u32 *resolved_ids = NULL; -++ u8 *visit_states = NULL; -++ -++ /* +1 for btf_void */ -++ resolved_sizes = kcalloc(nr_types + 1, sizeof(*resolved_sizes), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!resolved_sizes) -++ goto nomem; -++ -++ resolved_ids = kcalloc(nr_types + 1, sizeof(*resolved_ids), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!resolved_ids) -++ goto nomem; -++ -++ visit_states = kcalloc(nr_types + 1, sizeof(*visit_states), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!visit_states) -++ goto nomem; -++ -++ btf->resolved_sizes = resolved_sizes; -++ btf->resolved_ids = resolved_ids; -++ env->visit_states = visit_states; -++ -++ return 0; -++ -++nomem: -++ kvfree(resolved_sizes); -++ kvfree(resolved_ids); -++ kvfree(visit_states); -++ return -ENOMEM; -++} -++ -++static void btf_verifier_env_free(struct btf_verifier_env *env) -++{ -++ kvfree(env->visit_states); -++ kfree(env); -++} -++ -++static bool env_type_is_resolve_sink(const struct btf_verifier_env *env, -++ const struct btf_type *next_type) -++{ -++ switch (env->resolve_mode) { -++ case RESOLVE_TBD: -++ /* int, enum or void is a sink */ -++ return !btf_type_needs_resolve(next_type); -++ case RESOLVE_PTR: -++ /* int, enum, void, struct, array, func or func_proto is a sink -++ * for ptr -++ */ -++ return !btf_type_is_modifier(next_type) && -++ !btf_type_is_ptr(next_type); -++ case RESOLVE_STRUCT_OR_ARRAY: -++ /* int, enum, void, ptr, func or func_proto is a sink -++ * for struct and array -++ */ -++ return !btf_type_is_modifier(next_type) && -++ !btf_type_is_array(next_type) && -++ !btf_type_is_struct(next_type); -++ default: -++ BUG(); -++ } -++} -++ -++static bool env_type_is_resolved(const struct btf_verifier_env *env, -++ u32 type_id) -++{ -++ return env->visit_states[type_id] == RESOLVED; -++} -++ -++static int env_stack_push(struct btf_verifier_env *env, -++ const struct btf_type *t, u32 type_id) -++{ -++ struct resolve_vertex *v; -++ -++ if (env->top_stack == MAX_RESOLVE_DEPTH) -++ return -E2BIG; -++ -++ if (env->visit_states[type_id] != NOT_VISITED) -++ return -EEXIST; -++ -++ env->visit_states[type_id] = VISITED; -++ -++ v = &env->stack[env->top_stack++]; -++ v->t = t; -++ v->type_id = type_id; -++ v->next_member = 0; -++ -++ if (env->resolve_mode == RESOLVE_TBD) { -++ if (btf_type_is_ptr(t)) -++ env->resolve_mode = RESOLVE_PTR; -++ else if (btf_type_is_struct(t) || btf_type_is_array(t)) -++ env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY; -++ } -++ -++ return 0; -++} -++ -++static void env_stack_set_next_member(struct btf_verifier_env *env, -++ u16 next_member) -++{ -++ env->stack[env->top_stack - 1].next_member = next_member; -++} -++ -++static void env_stack_pop_resolved(struct btf_verifier_env *env, -++ u32 resolved_type_id, -++ u32 resolved_size) -++{ -++ u32 type_id = env->stack[--(env->top_stack)].type_id; -++ struct btf *btf = env->btf; -++ -++ btf->resolved_sizes[type_id] = resolved_size; -++ btf->resolved_ids[type_id] = resolved_type_id; -++ env->visit_states[type_id] = RESOLVED; -++} -++ -++static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env) -++{ -++ return env->top_stack ? &env->stack[env->top_stack - 1] : NULL; -++} -++ -++/* The input param "type_id" must point to a needs_resolve type */ -++static const struct btf_type *btf_type_id_resolve(const struct btf *btf, -++ u32 *type_id) -++{ -++ *type_id = btf->resolved_ids[*type_id]; -++ return btf_type_by_id(btf, *type_id); -++} -++ -++const struct btf_type *btf_type_id_size(const struct btf *btf, -++ u32 *type_id, u32 *ret_size) -++{ -++ const struct btf_type *size_type; -++ u32 size_type_id = *type_id; -++ u32 size = 0; -++ -++ size_type = btf_type_by_id(btf, size_type_id); -++ if (btf_type_nosize_or_null(size_type)) -++ return NULL; -++ -++ if (btf_type_has_size(size_type)) { -++ size = size_type->size; -++ } else if (btf_type_is_array(size_type)) { -++ size = btf->resolved_sizes[size_type_id]; -++ } else if (btf_type_is_ptr(size_type)) { -++ size = sizeof(void *); -++ } else { -++ if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) && -++ !btf_type_is_var(size_type))) -++ return NULL; -++ -++ size_type_id = btf->resolved_ids[size_type_id]; -++ size_type = btf_type_by_id(btf, size_type_id); -++ if (btf_type_nosize_or_null(size_type)) -++ return NULL; -++ else if (btf_type_has_size(size_type)) -++ size = size_type->size; -++ else if (btf_type_is_array(size_type)) -++ size = btf->resolved_sizes[size_type_id]; -++ else if (btf_type_is_ptr(size_type)) -++ size = sizeof(void *); -++ else -++ return NULL; -++ } -++ -++ *type_id = size_type_id; -++ if (ret_size) -++ *ret_size = size; -++ -++ return size_type; -++} -++ -++static int btf_df_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ btf_verifier_log_basic(env, struct_type, -++ "Unsupported check_member"); -++ return -EINVAL; -++} -++ -++static int btf_df_check_kflag_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ btf_verifier_log_basic(env, struct_type, -++ "Unsupported check_kflag_member"); -++ return -EINVAL; -++} -++ -++/* Used for ptr, array and struct/union type members. -++ * int, enum and modifier types have their specific callback functions. -++ */ -++static int btf_generic_check_kflag_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member bitfield_size"); -++ return -EINVAL; -++ } -++ -++ /* bitfield size is 0, so member->offset represents bit offset only. -++ * It is safe to call non kflag check_member variants. -++ */ -++ return btf_type_ops(member_type)->check_member(env, struct_type, -++ member, -++ member_type); -++} -++ -++static int btf_df_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ btf_verifier_log_basic(env, v->t, "Unsupported resolve"); -++ return -EINVAL; -++} -++ -++static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offsets, -++ struct seq_file *m) -++{ -++ seq_printf(m, "", BTF_INFO_KIND(t->info)); -++} -++ -++static int btf_int_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 int_data = btf_type_int(member_type); -++ u32 struct_bits_off = member->offset; -++ u32 struct_size = struct_type->size; -++ u32 nr_copy_bits; -++ u32 bytes_offset; -++ -++ if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "bits_offset exceeds U32_MAX"); -++ return -EINVAL; -++ } -++ -++ struct_bits_off += BTF_INT_OFFSET(int_data); -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ nr_copy_bits = BTF_INT_BITS(int_data) + -++ BITS_PER_BYTE_MASKED(struct_bits_off); -++ -++ if (nr_copy_bits > BITS_PER_U128) { -++ btf_verifier_log_member(env, struct_type, member, -++ "nr_copy_bits exceeds 128"); -++ return -EINVAL; -++ } -++ -++ if (struct_size < bytes_offset || -++ struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static int btf_int_check_kflag_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset; -++ u32 int_data = btf_type_int(member_type); -++ u32 struct_size = struct_type->size; -++ u32 nr_copy_bits; -++ -++ /* a regular int type is required for the kflag int member */ -++ if (!btf_type_int_is_regular(member_type)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member base type"); -++ return -EINVAL; -++ } -++ -++ /* check sanity of bitfield size */ -++ nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); -++ struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); -++ nr_int_data_bits = BTF_INT_BITS(int_data); -++ if (!nr_bits) { -++ /* Not a bitfield member, member offset must be at byte -++ * boundary. -++ */ -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member offset"); -++ return -EINVAL; -++ } -++ -++ nr_bits = nr_int_data_bits; -++ } else if (nr_bits > nr_int_data_bits) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member bitfield_size"); -++ return -EINVAL; -++ } -++ -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off); -++ if (nr_copy_bits > BITS_PER_U128) { -++ btf_verifier_log_member(env, struct_type, member, -++ "nr_copy_bits exceeds 128"); -++ return -EINVAL; -++ } -++ -++ if (struct_size < bytes_offset || -++ struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static s32 btf_int_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ u32 int_data, nr_bits, meta_needed = sizeof(int_data); -++ u16 encoding; -++ -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ int_data = btf_type_int(t); -++ if (int_data & ~BTF_INT_MASK) { -++ btf_verifier_log_basic(env, t, "Invalid int_data:%x", -++ int_data); -++ return -EINVAL; -++ } -++ -++ nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data); -++ -++ if (nr_bits > BITS_PER_U128) { -++ btf_verifier_log_type(env, t, "nr_bits exceeds %zu", -++ BITS_PER_U128); -++ return -EINVAL; -++ } -++ -++ if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) { -++ btf_verifier_log_type(env, t, "nr_bits exceeds type_size"); -++ return -EINVAL; -++ } -++ -++ /* -++ * Only one of the encoding bits is allowed and it -++ * should be sufficient for the pretty print purpose (i.e. decoding). -++ * Multiple bits can be allowed later if it is found -++ * to be insufficient. -++ */ -++ encoding = BTF_INT_ENCODING(int_data); -++ if (encoding && -++ encoding != BTF_INT_SIGNED && -++ encoding != BTF_INT_CHAR && -++ encoding != BTF_INT_BOOL) { -++ btf_verifier_log_type(env, t, "Unsupported encoding"); -++ return -ENOTSUPP; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return meta_needed; -++} -++ -++static void btf_int_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ int int_data = btf_type_int(t); -++ -++ btf_verifier_log(env, -++ "size=%u bits_offset=%u nr_bits=%u encoding=%s", -++ t->size, BTF_INT_OFFSET(int_data), -++ BTF_INT_BITS(int_data), -++ btf_int_encoding_str(BTF_INT_ENCODING(int_data))); -++} -++ -++static void btf_int128_print(struct seq_file *m, void *data) -++{ -++ /* data points to a __int128 number. -++ * Suppose -++ * int128_num = *(__int128 *)data; -++ * The below formulas shows what upper_num and lower_num represents: -++ * upper_num = int128_num >> 64; -++ * lower_num = int128_num & 0xffffffffFFFFFFFFULL; -++ */ -++ u64 upper_num, lower_num; -++ -++#ifdef __BIG_ENDIAN_BITFIELD -++ upper_num = *(u64 *)data; -++ lower_num = *(u64 *)(data + 8); -++#else -++ upper_num = *(u64 *)(data + 8); -++ lower_num = *(u64 *)data; -++#endif -++ if (upper_num == 0) -++ seq_printf(m, "0x%llx", lower_num); -++ else -++ seq_printf(m, "0x%llx%016llx", upper_num, lower_num); -++} -++ -++static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, -++ u16 right_shift_bits) -++{ -++ u64 upper_num, lower_num; -++ -++#ifdef __BIG_ENDIAN_BITFIELD -++ upper_num = print_num[0]; -++ lower_num = print_num[1]; -++#else -++ upper_num = print_num[1]; -++ lower_num = print_num[0]; -++#endif -++ -++ /* shake out un-needed bits by shift/or operations */ -++ if (left_shift_bits >= 64) { -++ upper_num = lower_num << (left_shift_bits - 64); -++ lower_num = 0; -++ } else { -++ upper_num = (upper_num << left_shift_bits) | -++ (lower_num >> (64 - left_shift_bits)); -++ lower_num = lower_num << left_shift_bits; -++ } -++ -++ if (right_shift_bits >= 64) { -++ lower_num = upper_num >> (right_shift_bits - 64); -++ upper_num = 0; -++ } else { -++ lower_num = (lower_num >> right_shift_bits) | -++ (upper_num << (64 - right_shift_bits)); -++ upper_num = upper_num >> right_shift_bits; -++ } -++ -++#ifdef __BIG_ENDIAN_BITFIELD -++ print_num[0] = upper_num; -++ print_num[1] = lower_num; -++#else -++ print_num[0] = lower_num; -++ print_num[1] = upper_num; -++#endif -++} -++ -++static void btf_bitfield_seq_show(void *data, u8 bits_offset, -++ u8 nr_bits, struct seq_file *m) -++{ -++ u16 left_shift_bits, right_shift_bits; -++ u8 nr_copy_bytes; -++ u8 nr_copy_bits; -++ u64 print_num[2] = {}; -++ -++ nr_copy_bits = nr_bits + bits_offset; -++ nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits); -++ -++ memcpy(print_num, data, nr_copy_bytes); -++ -++#ifdef __BIG_ENDIAN_BITFIELD -++ left_shift_bits = bits_offset; -++#else -++ left_shift_bits = BITS_PER_U128 - nr_copy_bits; -++#endif -++ right_shift_bits = BITS_PER_U128 - nr_bits; -++ -++ btf_int128_shift(print_num, left_shift_bits, right_shift_bits); -++ btf_int128_print(m, print_num); -++} -++ -++ -++static void btf_int_bits_seq_show(const struct btf *btf, -++ const struct btf_type *t, -++ void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ u32 int_data = btf_type_int(t); -++ u8 nr_bits = BTF_INT_BITS(int_data); -++ u8 total_bits_offset; -++ -++ /* -++ * bits_offset is at most 7. -++ * BTF_INT_OFFSET() cannot exceed 128 bits. -++ */ -++ total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data); -++ data += BITS_ROUNDDOWN_BYTES(total_bits_offset); -++ bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset); -++ btf_bitfield_seq_show(data, bits_offset, nr_bits, m); -++} -++ -++static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ u32 int_data = btf_type_int(t); -++ u8 encoding = BTF_INT_ENCODING(int_data); -++ bool sign = encoding & BTF_INT_SIGNED; -++ u8 nr_bits = BTF_INT_BITS(int_data); -++ -++ if (bits_offset || BTF_INT_OFFSET(int_data) || -++ BITS_PER_BYTE_MASKED(nr_bits)) { -++ btf_int_bits_seq_show(btf, t, data, bits_offset, m); -++ return; -++ } -++ -++ switch (nr_bits) { -++ case 128: -++ btf_int128_print(m, data); -++ break; -++ case 64: -++ if (sign) -++ seq_printf(m, "%lld", *(s64 *)data); -++ else -++ seq_printf(m, "%llu", *(u64 *)data); -++ break; -++ case 32: -++ if (sign) -++ seq_printf(m, "%d", *(s32 *)data); -++ else -++ seq_printf(m, "%u", *(u32 *)data); -++ break; -++ case 16: -++ if (sign) -++ seq_printf(m, "%d", *(s16 *)data); -++ else -++ seq_printf(m, "%u", *(u16 *)data); -++ break; -++ case 8: -++ if (sign) -++ seq_printf(m, "%d", *(s8 *)data); -++ else -++ seq_printf(m, "%u", *(u8 *)data); -++ break; -++ default: -++ btf_int_bits_seq_show(btf, t, data, bits_offset, m); -++ } -++} -++ -++static const struct btf_kind_operations int_ops = { -++ .check_meta = btf_int_check_meta, -++ .resolve = btf_df_resolve, -++ .check_member = btf_int_check_member, -++ .check_kflag_member = btf_int_check_kflag_member, -++ .log_details = btf_int_log, -++ .seq_show = btf_int_seq_show, -++}; -++ -++static int btf_modifier_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ const struct btf_type *resolved_type; -++ u32 resolved_type_id = member->type; -++ struct btf_member resolved_member; -++ struct btf *btf = env->btf; -++ -++ resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); -++ if (!resolved_type) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member"); -++ return -EINVAL; -++ } -++ -++ resolved_member = *member; -++ resolved_member.type = resolved_type_id; -++ -++ return btf_type_ops(resolved_type)->check_member(env, struct_type, -++ &resolved_member, -++ resolved_type); -++} -++ -++static int btf_modifier_check_kflag_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ const struct btf_type *resolved_type; -++ u32 resolved_type_id = member->type; -++ struct btf_member resolved_member; -++ struct btf *btf = env->btf; -++ -++ resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); -++ if (!resolved_type) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member"); -++ return -EINVAL; -++ } -++ -++ resolved_member = *member; -++ resolved_member.type = resolved_type_id; -++ -++ return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type, -++ &resolved_member, -++ resolved_type); -++} -++ -++static int btf_ptr_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_size, struct_bits_off, bytes_offset; -++ -++ struct_size = struct_type->size; -++ struct_bits_off = member->offset; -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member is not byte aligned"); -++ return -EINVAL; -++ } -++ -++ if (struct_size - bytes_offset < sizeof(void *)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static int btf_ref_type_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ if (!BTF_TYPE_ID_VALID(t->type)) { -++ btf_verifier_log_type(env, t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ /* typedef type must have a valid name, and other ref types, -++ * volatile, const, restrict, should have a null name. -++ */ -++ if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { -++ if (!t->name_off || -++ !btf_name_valid_identifier(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ } else { -++ if (t->name_off) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return 0; -++} -++ -++static int btf_modifier_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_type *t = v->t; -++ const struct btf_type *next_type; -++ u32 next_type_id = t->type; -++ struct btf *btf = env->btf; -++ -++ next_type = btf_type_by_id(btf, next_type_id); -++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, next_type) && -++ !env_type_is_resolved(env, next_type_id)) -++ return env_stack_push(env, next_type, next_type_id); -++ -++ /* Figure out the resolved next_type_id with size. -++ * They will be stored in the current modifier's -++ * resolved_ids and resolved_sizes such that it can -++ * save us a few type-following when we use it later (e.g. in -++ * pretty print). -++ */ -++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { -++ if (env_type_is_resolved(env, next_type_id)) -++ next_type = btf_type_id_resolve(btf, &next_type_id); -++ -++ /* "typedef void new_void", "const void"...etc */ -++ if (!btf_type_is_void(next_type) && -++ !btf_type_is_fwd(next_type) && -++ !btf_type_is_func_proto(next_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ } -++ -++ env_stack_pop_resolved(env, next_type_id, 0); -++ -++ return 0; -++} -++ -++static int btf_var_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_type *next_type; -++ const struct btf_type *t = v->t; -++ u32 next_type_id = t->type; -++ struct btf *btf = env->btf; -++ -++ next_type = btf_type_by_id(btf, next_type_id); -++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, next_type) && -++ !env_type_is_resolved(env, next_type_id)) -++ return env_stack_push(env, next_type, next_type_id); -++ -++ if (btf_type_is_modifier(next_type)) { -++ const struct btf_type *resolved_type; -++ u32 resolved_type_id; -++ -++ resolved_type_id = next_type_id; -++ resolved_type = btf_type_id_resolve(btf, &resolved_type_id); -++ -++ if (btf_type_is_ptr(resolved_type) && -++ !env_type_is_resolve_sink(env, resolved_type) && -++ !env_type_is_resolved(env, resolved_type_id)) -++ return env_stack_push(env, resolved_type, -++ resolved_type_id); -++ } -++ -++ /* We must resolve to something concrete at this point, no -++ * forward types or similar that would resolve to size of -++ * zero is allowed. -++ */ -++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ env_stack_pop_resolved(env, next_type_id, 0); -++ -++ return 0; -++} -++ -++static int btf_ptr_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_type *next_type; -++ const struct btf_type *t = v->t; -++ u32 next_type_id = t->type; -++ struct btf *btf = env->btf; -++ -++ next_type = btf_type_by_id(btf, next_type_id); -++ if (!next_type || btf_type_is_resolve_source_only(next_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, next_type) && -++ !env_type_is_resolved(env, next_type_id)) -++ return env_stack_push(env, next_type, next_type_id); -++ -++ /* If the modifier was RESOLVED during RESOLVE_STRUCT_OR_ARRAY, -++ * the modifier may have stopped resolving when it was resolved -++ * to a ptr (last-resolved-ptr). -++ * -++ * We now need to continue from the last-resolved-ptr to -++ * ensure the last-resolved-ptr will not referring back to -++ * the currenct ptr (t). -++ */ -++ if (btf_type_is_modifier(next_type)) { -++ const struct btf_type *resolved_type; -++ u32 resolved_type_id; -++ -++ resolved_type_id = next_type_id; -++ resolved_type = btf_type_id_resolve(btf, &resolved_type_id); -++ -++ if (btf_type_is_ptr(resolved_type) && -++ !env_type_is_resolve_sink(env, resolved_type) && -++ !env_type_is_resolved(env, resolved_type_id)) -++ return env_stack_push(env, resolved_type, -++ resolved_type_id); -++ } -++ -++ if (!btf_type_id_size(btf, &next_type_id, NULL)) { -++ if (env_type_is_resolved(env, next_type_id)) -++ next_type = btf_type_id_resolve(btf, &next_type_id); -++ -++ if (!btf_type_is_void(next_type) && -++ !btf_type_is_fwd(next_type) && -++ !btf_type_is_func_proto(next_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ } -++ -++ env_stack_pop_resolved(env, next_type_id, 0); -++ -++ return 0; -++} -++ -++static void btf_modifier_seq_show(const struct btf *btf, -++ const struct btf_type *t, -++ u32 type_id, void *data, -++ u8 bits_offset, struct seq_file *m) -++{ -++ t = btf_type_id_resolve(btf, &type_id); -++ -++ btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); -++} -++ -++static void btf_var_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ t = btf_type_id_resolve(btf, &type_id); -++ -++ btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m); -++} -++ -++static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ /* It is a hashed value */ -++ seq_printf(m, "%p", *(void **)data); -++} -++ -++static void btf_ref_type_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ btf_verifier_log(env, "type_id=%u", t->type); -++} -++ -++static struct btf_kind_operations modifier_ops = { -++ .check_meta = btf_ref_type_check_meta, -++ .resolve = btf_modifier_resolve, -++ .check_member = btf_modifier_check_member, -++ .check_kflag_member = btf_modifier_check_kflag_member, -++ .log_details = btf_ref_type_log, -++ .seq_show = btf_modifier_seq_show, -++}; -++ -++static struct btf_kind_operations ptr_ops = { -++ .check_meta = btf_ref_type_check_meta, -++ .resolve = btf_ptr_resolve, -++ .check_member = btf_ptr_check_member, -++ .check_kflag_member = btf_generic_check_kflag_member, -++ .log_details = btf_ref_type_log, -++ .seq_show = btf_ptr_seq_show, -++}; -++ -++static s32 btf_fwd_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (t->type) { -++ btf_verifier_log_type(env, t, "type != 0"); -++ return -EINVAL; -++ } -++ -++ /* fwd type must have a valid name */ -++ if (!t->name_off || -++ !btf_name_valid_identifier(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return 0; -++} -++ -++static void btf_fwd_type_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct"); -++} -++ -++static struct btf_kind_operations fwd_ops = { -++ .check_meta = btf_fwd_check_meta, -++ .resolve = btf_df_resolve, -++ .check_member = btf_df_check_member, -++ .check_kflag_member = btf_df_check_kflag_member, -++ .log_details = btf_fwd_type_log, -++ .seq_show = btf_df_seq_show, -++}; -++ -++static int btf_array_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_bits_off = member->offset; -++ u32 struct_size, bytes_offset; -++ u32 array_type_id, array_size; -++ struct btf *btf = env->btf; -++ -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member is not byte aligned"); -++ return -EINVAL; -++ } -++ -++ array_type_id = member->type; -++ btf_type_id_size(btf, &array_type_id, &array_size); -++ struct_size = struct_type->size; -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ if (struct_size - bytes_offset < array_size) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static s32 btf_array_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ const struct btf_array *array = btf_type_array(t); -++ u32 meta_needed = sizeof(*array); -++ -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ /* array type should not have a name */ -++ if (t->name_off) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ if (t->size) { -++ btf_verifier_log_type(env, t, "size != 0"); -++ return -EINVAL; -++ } -++ -++ /* Array elem type and index type cannot be in type void, -++ * so !array->type and !array->index_type are not allowed. -++ */ -++ if (!array->type || !BTF_TYPE_ID_VALID(array->type)) { -++ btf_verifier_log_type(env, t, "Invalid elem"); -++ return -EINVAL; -++ } -++ -++ if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) { -++ btf_verifier_log_type(env, t, "Invalid index"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return meta_needed; -++} -++ -++static int btf_array_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_array *array = btf_type_array(v->t); -++ const struct btf_type *elem_type, *index_type; -++ u32 elem_type_id, index_type_id; -++ struct btf *btf = env->btf; -++ u32 elem_size; -++ -++ /* Check array->index_type */ -++ index_type_id = array->index_type; -++ index_type = btf_type_by_id(btf, index_type_id); -++ if (btf_type_nosize_or_null(index_type) || -++ btf_type_is_resolve_source_only(index_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid index"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, index_type) && -++ !env_type_is_resolved(env, index_type_id)) -++ return env_stack_push(env, index_type, index_type_id); -++ -++ index_type = btf_type_id_size(btf, &index_type_id, NULL); -++ if (!index_type || !btf_type_is_int(index_type) || -++ !btf_type_int_is_regular(index_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid index"); -++ return -EINVAL; -++ } -++ -++ /* Check array->type */ -++ elem_type_id = array->type; -++ elem_type = btf_type_by_id(btf, elem_type_id); -++ if (btf_type_nosize_or_null(elem_type) || -++ btf_type_is_resolve_source_only(elem_type)) { -++ btf_verifier_log_type(env, v->t, -++ "Invalid elem"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, elem_type) && -++ !env_type_is_resolved(env, elem_type_id)) -++ return env_stack_push(env, elem_type, elem_type_id); -++ -++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); -++ if (!elem_type) { -++ btf_verifier_log_type(env, v->t, "Invalid elem"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) { -++ btf_verifier_log_type(env, v->t, "Invalid array of int"); -++ return -EINVAL; -++ } -++ -++ if (array->nelems && elem_size > U32_MAX / array->nelems) { -++ btf_verifier_log_type(env, v->t, -++ "Array size overflows U32_MAX"); -++ return -EINVAL; -++ } -++ -++ env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems); -++ -++ return 0; -++} -++ -++static void btf_array_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ const struct btf_array *array = btf_type_array(t); -++ -++ btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u", -++ array->type, array->index_type, array->nelems); -++} -++ -++static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ const struct btf_array *array = btf_type_array(t); -++ const struct btf_kind_operations *elem_ops; -++ const struct btf_type *elem_type; -++ u32 i, elem_size, elem_type_id; -++ -++ elem_type_id = array->type; -++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); -++ elem_ops = btf_type_ops(elem_type); -++ seq_puts(m, "["); -++ for (i = 0; i < array->nelems; i++) { -++ if (i) -++ seq_puts(m, ","); -++ -++ elem_ops->seq_show(btf, elem_type, elem_type_id, data, -++ bits_offset, m); -++ data += elem_size; -++ } -++ seq_puts(m, "]"); -++} -++ -++static struct btf_kind_operations array_ops = { -++ .check_meta = btf_array_check_meta, -++ .resolve = btf_array_resolve, -++ .check_member = btf_array_check_member, -++ .check_kflag_member = btf_generic_check_kflag_member, -++ .log_details = btf_array_log, -++ .seq_show = btf_array_seq_show, -++}; -++ -++static int btf_struct_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_bits_off = member->offset; -++ u32 struct_size, bytes_offset; -++ -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member is not byte aligned"); -++ return -EINVAL; -++ } -++ -++ struct_size = struct_type->size; -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ if (struct_size - bytes_offset < member_type->size) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static s32 btf_struct_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION; -++ const struct btf_member *member; -++ u32 meta_needed, last_offset; -++ struct btf *btf = env->btf; -++ u32 struct_size = t->size; -++ u32 offset; -++ u16 i; -++ -++ meta_needed = btf_type_vlen(t) * sizeof(*member); -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ /* struct type either no name or a valid one */ -++ if (t->name_off && -++ !btf_name_valid_identifier(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ last_offset = 0; -++ for_each_member(i, t, member) { -++ if (!btf_name_offset_valid(btf, member->name_off)) { -++ btf_verifier_log_member(env, t, member, -++ "Invalid member name_offset:%u", -++ member->name_off); -++ return -EINVAL; -++ } -++ -++ /* struct member either no name or a valid one */ -++ if (member->name_off && -++ !btf_name_valid_identifier(btf, member->name_off)) { -++ btf_verifier_log_member(env, t, member, "Invalid name"); -++ return -EINVAL; -++ } -++ /* A member cannot be in type void */ -++ if (!member->type || !BTF_TYPE_ID_VALID(member->type)) { -++ btf_verifier_log_member(env, t, member, -++ "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ offset = btf_member_bit_offset(t, member); -++ if (is_union && offset) { -++ btf_verifier_log_member(env, t, member, -++ "Invalid member bits_offset"); -++ return -EINVAL; -++ } -++ -++ /* -++ * ">" instead of ">=" because the last member could be -++ * "char a[0];" -++ */ -++ if (last_offset > offset) { -++ btf_verifier_log_member(env, t, member, -++ "Invalid member bits_offset"); -++ return -EINVAL; -++ } -++ -++ if (BITS_ROUNDUP_BYTES(offset) > struct_size) { -++ btf_verifier_log_member(env, t, member, -++ "Member bits_offset exceeds its struct size"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_member(env, t, member, NULL); -++ last_offset = offset; -++ } -++ -++ return meta_needed; -++} -++ -++static int btf_struct_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_member *member; -++ int err; -++ u16 i; -++ -++ /* Before continue resolving the next_member, -++ * ensure the last member is indeed resolved to a -++ * type with size info. -++ */ -++ if (v->next_member) { -++ const struct btf_type *last_member_type; -++ const struct btf_member *last_member; -++ u16 last_member_type_id; -++ -++ last_member = btf_type_member(v->t) + v->next_member - 1; -++ last_member_type_id = last_member->type; -++ if (WARN_ON_ONCE(!env_type_is_resolved(env, -++ last_member_type_id))) -++ return -EINVAL; -++ -++ last_member_type = btf_type_by_id(env->btf, -++ last_member_type_id); -++ if (btf_type_kflag(v->t)) -++ err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t, -++ last_member, -++ last_member_type); -++ else -++ err = btf_type_ops(last_member_type)->check_member(env, v->t, -++ last_member, -++ last_member_type); -++ if (err) -++ return err; -++ } -++ -++ for_each_member_from(i, v->next_member, v->t, member) { -++ u32 member_type_id = member->type; -++ const struct btf_type *member_type = btf_type_by_id(env->btf, -++ member_type_id); -++ -++ if (btf_type_nosize_or_null(member_type) || -++ btf_type_is_resolve_source_only(member_type)) { -++ btf_verifier_log_member(env, v->t, member, -++ "Invalid member"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, member_type) && -++ !env_type_is_resolved(env, member_type_id)) { -++ env_stack_set_next_member(env, i + 1); -++ return env_stack_push(env, member_type, member_type_id); -++ } -++ -++ if (btf_type_kflag(v->t)) -++ err = btf_type_ops(member_type)->check_kflag_member(env, v->t, -++ member, -++ member_type); -++ else -++ err = btf_type_ops(member_type)->check_member(env, v->t, -++ member, -++ member_type); -++ if (err) -++ return err; -++ } -++ -++ env_stack_pop_resolved(env, 0, 0); -++ -++ return 0; -++} -++ -++static void btf_struct_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); -++} -++ -++/* find 'struct bpf_spin_lock' in map value. -++ * return >= 0 offset if found -++ * and < 0 in case of error -++ */ -++int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t) -++{ -++ const struct btf_member *member; -++ u32 i, off = -ENOENT; -++ -++ if (!__btf_type_is_struct(t)) -++ return -EINVAL; -++ -++ for_each_member(i, t, member) { -++ const struct btf_type *member_type = btf_type_by_id(btf, -++ member->type); -++ if (!__btf_type_is_struct(member_type)) -++ continue; -++ if (member_type->size != sizeof(struct bpf_spin_lock)) -++ continue; -++ if (strcmp(__btf_name_by_offset(btf, member_type->name_off), -++ "bpf_spin_lock")) -++ continue; -++ if (off != -ENOENT) -++ /* only one 'struct bpf_spin_lock' is allowed */ -++ return -E2BIG; -++ off = btf_member_bit_offset(t, member); -++ if (off % 8) -++ /* valid C code cannot generate such BTF */ -++ return -EINVAL; -++ off /= 8; -++ if (off % __alignof__(struct bpf_spin_lock)) -++ /* valid struct bpf_spin_lock will be 4 byte aligned */ -++ return -EINVAL; -++ } -++ return off; -++} -++ -++static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ","; -++ const struct btf_member *member; -++ u32 i; -++ -++ seq_puts(m, "{"); -++ for_each_member(i, t, member) { -++ const struct btf_type *member_type = btf_type_by_id(btf, -++ member->type); -++ const struct btf_kind_operations *ops; -++ u32 member_offset, bitfield_size; -++ u32 bytes_offset; -++ u8 bits8_offset; -++ -++ if (i) -++ seq_puts(m, seq); -++ -++ member_offset = btf_member_bit_offset(t, member); -++ bitfield_size = btf_member_bitfield_size(t, member); -++ bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset); -++ bits8_offset = BITS_PER_BYTE_MASKED(member_offset); -++ if (bitfield_size) { -++ btf_bitfield_seq_show(data + bytes_offset, bits8_offset, -++ bitfield_size, m); -++ } else { -++ ops = btf_type_ops(member_type); -++ ops->seq_show(btf, member_type, member->type, -++ data + bytes_offset, bits8_offset, m); -++ } -++ } -++ seq_puts(m, "}"); -++} -++ -++static struct btf_kind_operations struct_ops = { -++ .check_meta = btf_struct_check_meta, -++ .resolve = btf_struct_resolve, -++ .check_member = btf_struct_check_member, -++ .check_kflag_member = btf_generic_check_kflag_member, -++ .log_details = btf_struct_log, -++ .seq_show = btf_struct_seq_show, -++}; -++ -++static int btf_enum_check_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_bits_off = member->offset; -++ u32 struct_size, bytes_offset; -++ -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member is not byte aligned"); -++ return -EINVAL; -++ } -++ -++ struct_size = struct_type->size; -++ bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off); -++ if (struct_size - bytes_offset < member_type->size) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static int btf_enum_check_kflag_member(struct btf_verifier_env *env, -++ const struct btf_type *struct_type, -++ const struct btf_member *member, -++ const struct btf_type *member_type) -++{ -++ u32 struct_bits_off, nr_bits, bytes_end, struct_size; -++ u32 int_bitsize = sizeof(int) * BITS_PER_BYTE; -++ -++ struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset); -++ nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset); -++ if (!nr_bits) { -++ if (BITS_PER_BYTE_MASKED(struct_bits_off)) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member is not byte aligned"); -++ return -EINVAL; -++ } -++ -++ nr_bits = int_bitsize; -++ } else if (nr_bits > int_bitsize) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Invalid member bitfield_size"); -++ return -EINVAL; -++ } -++ -++ struct_size = struct_type->size; -++ bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits); -++ if (struct_size < bytes_end) { -++ btf_verifier_log_member(env, struct_type, member, -++ "Member exceeds struct_size"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static s32 btf_enum_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ const struct btf_enum *enums = btf_type_enum(t); -++ struct btf *btf = env->btf; -++ u16 i, nr_enums; -++ u32 meta_needed; -++ -++ nr_enums = btf_type_vlen(t); -++ meta_needed = nr_enums * sizeof(*enums); -++ -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ if (t->size > 8 || !is_power_of_2(t->size)) { -++ btf_verifier_log_type(env, t, "Unexpected size"); -++ return -EINVAL; -++ } -++ -++ /* enum type either no name or a valid one */ -++ if (t->name_off && -++ !btf_name_valid_identifier(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ for (i = 0; i < nr_enums; i++) { -++ if (!btf_name_offset_valid(btf, enums[i].name_off)) { -++ btf_verifier_log(env, "\tInvalid name_offset:%u", -++ enums[i].name_off); -++ return -EINVAL; -++ } -++ -++ /* enum member must have a valid name */ -++ if (!enums[i].name_off || -++ !btf_name_valid_identifier(btf, enums[i].name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ -++ btf_verifier_log(env, "\t%s val=%d\n", -++ __btf_name_by_offset(btf, enums[i].name_off), -++ enums[i].val); -++ } -++ -++ return meta_needed; -++} -++ -++static void btf_enum_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); -++} -++ -++static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t, -++ u32 type_id, void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ const struct btf_enum *enums = btf_type_enum(t); -++ u32 i, nr_enums = btf_type_vlen(t); -++ int v = *(int *)data; -++ -++ for (i = 0; i < nr_enums; i++) { -++ if (v == enums[i].val) { -++ seq_printf(m, "%s", -++ __btf_name_by_offset(btf, -++ enums[i].name_off)); -++ return; -++ } -++ } -++ -++ seq_printf(m, "%d", v); -++} -++ -++static struct btf_kind_operations enum_ops = { -++ .check_meta = btf_enum_check_meta, -++ .resolve = btf_df_resolve, -++ .check_member = btf_enum_check_member, -++ .check_kflag_member = btf_enum_check_kflag_member, -++ .log_details = btf_enum_log, -++ .seq_show = btf_enum_seq_show, -++}; -++ -++static s32 btf_func_proto_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param); -++ -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ if (t->name_off) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return meta_needed; -++} -++ -++static void btf_func_proto_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ const struct btf_param *args = (const struct btf_param *)(t + 1); -++ u16 nr_args = btf_type_vlen(t), i; -++ -++ btf_verifier_log(env, "return=%u args=(", t->type); -++ if (!nr_args) { -++ btf_verifier_log(env, "void"); -++ goto done; -++ } -++ -++ if (nr_args == 1 && !args[0].type) { -++ /* Only one vararg */ -++ btf_verifier_log(env, "vararg"); -++ goto done; -++ } -++ -++ btf_verifier_log(env, "%u %s", args[0].type, -++ __btf_name_by_offset(env->btf, -++ args[0].name_off)); -++ for (i = 1; i < nr_args - 1; i++) -++ btf_verifier_log(env, ", %u %s", args[i].type, -++ __btf_name_by_offset(env->btf, -++ args[i].name_off)); -++ -++ if (nr_args > 1) { -++ const struct btf_param *last_arg = &args[nr_args - 1]; -++ -++ if (last_arg->type) -++ btf_verifier_log(env, ", %u %s", last_arg->type, -++ __btf_name_by_offset(env->btf, -++ last_arg->name_off)); -++ else -++ btf_verifier_log(env, ", vararg"); -++ } -++ -++done: -++ btf_verifier_log(env, ")"); -++} -++ -++static struct btf_kind_operations func_proto_ops = { -++ .check_meta = btf_func_proto_check_meta, -++ .resolve = btf_df_resolve, -++ /* -++ * BTF_KIND_FUNC_PROTO cannot be directly referred by -++ * a struct's member. -++ * -++ * It should be a funciton pointer instead. -++ * (i.e. struct's member -> BTF_KIND_PTR -> BTF_KIND_FUNC_PROTO) -++ * -++ * Hence, there is no btf_func_check_member(). -++ */ -++ .check_member = btf_df_check_member, -++ .check_kflag_member = btf_df_check_kflag_member, -++ .log_details = btf_func_proto_log, -++ .seq_show = btf_df_seq_show, -++}; -++ -++static s32 btf_func_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ if (!t->name_off || -++ !btf_name_valid_identifier(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return 0; -++} -++ -++static struct btf_kind_operations func_ops = { -++ .check_meta = btf_func_check_meta, -++ .resolve = btf_df_resolve, -++ .check_member = btf_df_check_member, -++ .check_kflag_member = btf_df_check_kflag_member, -++ .log_details = btf_ref_type_log, -++ .seq_show = btf_df_seq_show, -++}; -++ -++static s32 btf_var_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ const struct btf_var *var; -++ u32 meta_needed = sizeof(*var); -++ -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ if (btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen != 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ if (!t->name_off || -++ !__btf_name_valid(env->btf, t->name_off, true)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ /* A var cannot be in type void */ -++ if (!t->type || !BTF_TYPE_ID_VALID(t->type)) { -++ btf_verifier_log_type(env, t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ var = btf_type_var(t); -++ if (var->linkage != BTF_VAR_STATIC && -++ var->linkage != BTF_VAR_GLOBAL_ALLOCATED) { -++ btf_verifier_log_type(env, t, "Linkage not supported"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ return meta_needed; -++} -++ -++static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t) -++{ -++ const struct btf_var *var = btf_type_var(t); -++ -++ btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage); -++} -++ -++static const struct btf_kind_operations var_ops = { -++ .check_meta = btf_var_check_meta, -++ .resolve = btf_var_resolve, -++ .check_member = btf_df_check_member, -++ .check_kflag_member = btf_df_check_kflag_member, -++ .log_details = btf_var_log, -++ .seq_show = btf_var_seq_show, -++}; -++ -++static s32 btf_datasec_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ const struct btf_var_secinfo *vsi; -++ u64 last_vsi_end_off = 0, sum = 0; -++ u32 i, meta_needed; -++ -++ meta_needed = btf_type_vlen(t) * sizeof(*vsi); -++ if (meta_left < meta_needed) { -++ btf_verifier_log_basic(env, t, -++ "meta_left:%u meta_needed:%u", -++ meta_left, meta_needed); -++ return -EINVAL; -++ } -++ -++ if (!btf_type_vlen(t)) { -++ btf_verifier_log_type(env, t, "vlen == 0"); -++ return -EINVAL; -++ } -++ -++ if (!t->size) { -++ btf_verifier_log_type(env, t, "size == 0"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_kflag(t)) { -++ btf_verifier_log_type(env, t, "Invalid btf_info kind_flag"); -++ return -EINVAL; -++ } -++ -++ if (!t->name_off || -++ !btf_name_valid_section(env->btf, t->name_off)) { -++ btf_verifier_log_type(env, t, "Invalid name"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_type(env, t, NULL); -++ -++ for_each_vsi(i, t, vsi) { -++ /* A var cannot be in type void */ -++ if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) { -++ btf_verifier_log_vsi(env, t, vsi, -++ "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) { -++ btf_verifier_log_vsi(env, t, vsi, -++ "Invalid offset"); -++ return -EINVAL; -++ } -++ -++ if (!vsi->size || vsi->size > t->size) { -++ btf_verifier_log_vsi(env, t, vsi, -++ "Invalid size"); -++ return -EINVAL; -++ } -++ -++ last_vsi_end_off = vsi->offset + vsi->size; -++ if (last_vsi_end_off > t->size) { -++ btf_verifier_log_vsi(env, t, vsi, -++ "Invalid offset+size"); -++ return -EINVAL; -++ } -++ -++ btf_verifier_log_vsi(env, t, vsi, NULL); -++ sum += vsi->size; -++ } -++ -++ if (t->size < sum) { -++ btf_verifier_log_type(env, t, "Invalid btf_info size"); -++ return -EINVAL; -++ } -++ -++ return meta_needed; -++} -++ -++static int btf_datasec_resolve(struct btf_verifier_env *env, -++ const struct resolve_vertex *v) -++{ -++ const struct btf_var_secinfo *vsi; -++ struct btf *btf = env->btf; -++ u16 i; -++ -++ for_each_vsi_from(i, v->next_member, v->t, vsi) { -++ u32 var_type_id = vsi->type, type_id, type_size = 0; -++ const struct btf_type *var_type = btf_type_by_id(env->btf, -++ var_type_id); -++ if (!var_type || !btf_type_is_var(var_type)) { -++ btf_verifier_log_vsi(env, v->t, vsi, -++ "Not a VAR kind member"); -++ return -EINVAL; -++ } -++ -++ if (!env_type_is_resolve_sink(env, var_type) && -++ !env_type_is_resolved(env, var_type_id)) { -++ env_stack_set_next_member(env, i + 1); -++ return env_stack_push(env, var_type, var_type_id); -++ } -++ -++ type_id = var_type->type; -++ if (!btf_type_id_size(btf, &type_id, &type_size)) { -++ btf_verifier_log_vsi(env, v->t, vsi, "Invalid type"); -++ return -EINVAL; -++ } -++ -++ if (vsi->size < type_size) { -++ btf_verifier_log_vsi(env, v->t, vsi, "Invalid size"); -++ return -EINVAL; -++ } -++ } -++ -++ env_stack_pop_resolved(env, 0, 0); -++ return 0; -++} -++ -++static void btf_datasec_log(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t)); -++} -++ -++static void btf_datasec_seq_show(const struct btf *btf, -++ const struct btf_type *t, u32 type_id, -++ void *data, u8 bits_offset, -++ struct seq_file *m) -++{ -++ const struct btf_var_secinfo *vsi; -++ const struct btf_type *var; -++ u32 i; -++ -++ seq_printf(m, "section (\"%s\") = {", __btf_name_by_offset(btf, t->name_off)); -++ for_each_vsi(i, t, vsi) { -++ var = btf_type_by_id(btf, vsi->type); -++ if (i) -++ seq_puts(m, ","); -++ btf_type_ops(var)->seq_show(btf, var, vsi->type, -++ data + vsi->offset, bits_offset, m); -++ } -++ seq_puts(m, "}"); -++} -++ -++static const struct btf_kind_operations datasec_ops = { -++ .check_meta = btf_datasec_check_meta, -++ .resolve = btf_datasec_resolve, -++ .check_member = btf_df_check_member, -++ .check_kflag_member = btf_df_check_kflag_member, -++ .log_details = btf_datasec_log, -++ .seq_show = btf_datasec_seq_show, -++}; -++ -++static int btf_func_proto_check(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ const struct btf_type *ret_type; -++ const struct btf_param *args; -++ const struct btf *btf; -++ u16 nr_args, i; -++ int err; -++ -++ btf = env->btf; -++ args = (const struct btf_param *)(t + 1); -++ nr_args = btf_type_vlen(t); -++ -++ /* Check func return type which could be "void" (t->type == 0) */ -++ if (t->type) { -++ u32 ret_type_id = t->type; -++ -++ ret_type = btf_type_by_id(btf, ret_type_id); -++ if (!ret_type) { -++ btf_verifier_log_type(env, t, "Invalid return type"); -++ return -EINVAL; -++ } -++ -++ if (btf_type_needs_resolve(ret_type) && -++ !env_type_is_resolved(env, ret_type_id)) { -++ err = btf_resolve(env, ret_type, ret_type_id); -++ if (err) -++ return err; -++ } -++ -++ /* Ensure the return type is a type that has a size */ -++ if (!btf_type_id_size(btf, &ret_type_id, NULL)) { -++ btf_verifier_log_type(env, t, "Invalid return type"); -++ return -EINVAL; -++ } -++ } -++ -++ if (!nr_args) -++ return 0; -++ -++ /* Last func arg type_id could be 0 if it is a vararg */ -++ if (!args[nr_args - 1].type) { -++ if (args[nr_args - 1].name_off) { -++ btf_verifier_log_type(env, t, "Invalid arg#%u", -++ nr_args); -++ return -EINVAL; -++ } -++ nr_args--; -++ } -++ -++ err = 0; -++ for (i = 0; i < nr_args; i++) { -++ const struct btf_type *arg_type; -++ u32 arg_type_id; -++ -++ arg_type_id = args[i].type; -++ arg_type = btf_type_by_id(btf, arg_type_id); -++ if (!arg_type) { -++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); -++ err = -EINVAL; -++ break; -++ } -++ -++ if (args[i].name_off && -++ (!btf_name_offset_valid(btf, args[i].name_off) || -++ !btf_name_valid_identifier(btf, args[i].name_off))) { -++ btf_verifier_log_type(env, t, -++ "Invalid arg#%u", i + 1); -++ err = -EINVAL; -++ break; -++ } -++ -++ if (btf_type_needs_resolve(arg_type) && -++ !env_type_is_resolved(env, arg_type_id)) { -++ err = btf_resolve(env, arg_type, arg_type_id); -++ if (err) -++ break; -++ } -++ -++ if (!btf_type_id_size(btf, &arg_type_id, NULL)) { -++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); -++ err = -EINVAL; -++ break; -++ } -++ } -++ -++ return err; -++} -++ -++static int btf_func_check(struct btf_verifier_env *env, -++ const struct btf_type *t) -++{ -++ const struct btf_type *proto_type; -++ const struct btf_param *args; -++ const struct btf *btf; -++ u16 nr_args, i; -++ -++ btf = env->btf; -++ proto_type = btf_type_by_id(btf, t->type); -++ -++ if (!proto_type || !btf_type_is_func_proto(proto_type)) { -++ btf_verifier_log_type(env, t, "Invalid type_id"); -++ return -EINVAL; -++ } -++ -++ args = (const struct btf_param *)(proto_type + 1); -++ nr_args = btf_type_vlen(proto_type); -++ for (i = 0; i < nr_args; i++) { -++ if (!args[i].name_off && args[i].type) { -++ btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1); -++ return -EINVAL; -++ } -++ } -++ -++ return 0; -++} -++ -++static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { -++ [BTF_KIND_INT] = &int_ops, -++ [BTF_KIND_PTR] = &ptr_ops, -++ [BTF_KIND_ARRAY] = &array_ops, -++ [BTF_KIND_STRUCT] = &struct_ops, -++ [BTF_KIND_UNION] = &struct_ops, -++ [BTF_KIND_ENUM] = &enum_ops, -++ [BTF_KIND_FWD] = &fwd_ops, -++ [BTF_KIND_TYPEDEF] = &modifier_ops, -++ [BTF_KIND_VOLATILE] = &modifier_ops, -++ [BTF_KIND_CONST] = &modifier_ops, -++ [BTF_KIND_RESTRICT] = &modifier_ops, -++ [BTF_KIND_FUNC] = &func_ops, -++ [BTF_KIND_FUNC_PROTO] = &func_proto_ops, -++ [BTF_KIND_VAR] = &var_ops, -++ [BTF_KIND_DATASEC] = &datasec_ops, -++}; -++ -++static s32 btf_check_meta(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 meta_left) -++{ -++ u32 saved_meta_left = meta_left; -++ s32 var_meta_size; -++ -++ if (meta_left < sizeof(*t)) { -++ btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu", -++ env->log_type_id, meta_left, sizeof(*t)); -++ return -EINVAL; -++ } -++ meta_left -= sizeof(*t); -++ -++ if (t->info & ~BTF_INFO_MASK) { -++ btf_verifier_log(env, "[%u] Invalid btf_info:%x", -++ env->log_type_id, t->info); -++ return -EINVAL; -++ } -++ -++ if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX || -++ BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) { -++ btf_verifier_log(env, "[%u] Invalid kind:%u", -++ env->log_type_id, BTF_INFO_KIND(t->info)); -++ return -EINVAL; -++ } -++ -++ if (!btf_name_offset_valid(env->btf, t->name_off)) { -++ btf_verifier_log(env, "[%u] Invalid name_offset:%u", -++ env->log_type_id, t->name_off); -++ return -EINVAL; -++ } -++ -++ var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left); -++ if (var_meta_size < 0) -++ return var_meta_size; -++ -++ meta_left -= var_meta_size; -++ -++ return saved_meta_left - meta_left; -++} -++ -++static int btf_check_all_metas(struct btf_verifier_env *env) -++{ -++ struct btf *btf = env->btf; -++ struct btf_header *hdr; -++ void *cur, *end; -++ -++ hdr = &btf->hdr; -++ cur = btf->nohdr_data + hdr->type_off; -++ end = cur + hdr->type_len; -++ -++ env->log_type_id = 1; -++ while (cur < end) { -++ struct btf_type *t = cur; -++ s32 meta_size; -++ -++ meta_size = btf_check_meta(env, t, end - cur); -++ if (meta_size < 0) -++ return meta_size; -++ -++ btf_add_type(env, t); -++ cur += meta_size; -++ env->log_type_id++; -++ } -++ -++ return 0; -++} -++ -++static bool btf_resolve_valid(struct btf_verifier_env *env, -++ const struct btf_type *t, -++ u32 type_id) -++{ -++ struct btf *btf = env->btf; -++ -++ if (!env_type_is_resolved(env, type_id)) -++ return false; -++ -++ if (btf_type_is_struct(t) || btf_type_is_datasec(t)) -++ return !btf->resolved_ids[type_id] && -++ !btf->resolved_sizes[type_id]; -++ -++ if (btf_type_is_modifier(t) || btf_type_is_ptr(t) || -++ btf_type_is_var(t)) { -++ t = btf_type_id_resolve(btf, &type_id); -++ return t && -++ !btf_type_is_modifier(t) && -++ !btf_type_is_var(t) && -++ !btf_type_is_datasec(t); -++ } -++ -++ if (btf_type_is_array(t)) { -++ const struct btf_array *array = btf_type_array(t); -++ const struct btf_type *elem_type; -++ u32 elem_type_id = array->type; -++ u32 elem_size; -++ -++ elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); -++ return elem_type && !btf_type_is_modifier(elem_type) && -++ (array->nelems * elem_size == -++ btf->resolved_sizes[type_id]); -++ } -++ -++ return false; -++} -++ -++static int btf_resolve(struct btf_verifier_env *env, -++ const struct btf_type *t, u32 type_id) -++{ -++ u32 save_log_type_id = env->log_type_id; -++ const struct resolve_vertex *v; -++ int err = 0; -++ -++ env->resolve_mode = RESOLVE_TBD; -++ env_stack_push(env, t, type_id); -++ while (!err && (v = env_stack_peak(env))) { -++ env->log_type_id = v->type_id; -++ err = btf_type_ops(v->t)->resolve(env, v); -++ } -++ -++ env->log_type_id = type_id; -++ if (err == -E2BIG) { -++ btf_verifier_log_type(env, t, -++ "Exceeded max resolving depth:%u", -++ MAX_RESOLVE_DEPTH); -++ } else if (err == -EEXIST) { -++ btf_verifier_log_type(env, t, "Loop detected"); -++ } -++ -++ /* Final sanity check */ -++ if (!err && !btf_resolve_valid(env, t, type_id)) { -++ btf_verifier_log_type(env, t, "Invalid resolve state"); -++ err = -EINVAL; -++ } -++ -++ env->log_type_id = save_log_type_id; -++ return err; -++} -++ -++static int btf_check_all_types(struct btf_verifier_env *env) -++{ -++ struct btf *btf = env->btf; -++ u32 type_id; -++ int err; -++ -++ err = env_resolve_init(env); -++ if (err) -++ return err; -++ -++ env->phase++; -++ for (type_id = 1; type_id <= btf->nr_types; type_id++) { -++ const struct btf_type *t = btf_type_by_id(btf, type_id); -++ -++ env->log_type_id = type_id; -++ if (btf_type_needs_resolve(t) && -++ !env_type_is_resolved(env, type_id)) { -++ err = btf_resolve(env, t, type_id); -++ if (err) -++ return err; -++ } -++ -++ if (btf_type_is_func_proto(t)) { -++ err = btf_func_proto_check(env, t); -++ if (err) -++ return err; -++ } -++ -++ if (btf_type_is_func(t)) { -++ err = btf_func_check(env, t); -++ if (err) -++ return err; -++ } -++ } -++ -++ return 0; -++} -++ -++static int btf_parse_type_sec(struct btf_verifier_env *env) -++{ -++ const struct btf_header *hdr = &env->btf->hdr; -++ int err; -++ -++ /* Type section must align to 4 bytes */ -++ if (hdr->type_off & (sizeof(u32) - 1)) { -++ btf_verifier_log(env, "Unaligned type_off"); -++ return -EINVAL; -++ } -++ -++ if (!hdr->type_len) { -++ btf_verifier_log(env, "No type found"); -++ return -EINVAL; -++ } -++ -++ err = btf_check_all_metas(env); -++ if (err) -++ return err; -++ -++ return btf_check_all_types(env); -++} -++ -++static int btf_parse_str_sec(struct btf_verifier_env *env) -++{ -++ const struct btf_header *hdr; -++ struct btf *btf = env->btf; -++ const char *start, *end; -++ -++ hdr = &btf->hdr; -++ start = btf->nohdr_data + hdr->str_off; -++ end = start + hdr->str_len; -++ -++ if (end != btf->data + btf->data_size) { -++ btf_verifier_log(env, "String section is not at the end"); -++ return -EINVAL; -++ } -++ -++ if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || -++ start[0] || end[-1]) { -++ btf_verifier_log(env, "Invalid string section"); -++ return -EINVAL; -++ } -++ -++ btf->strings = start; -++ -++ return 0; -++} -++ -++static const size_t btf_sec_info_offset[] = { -++ offsetof(struct btf_header, type_off), -++ offsetof(struct btf_header, str_off), -++}; -++ -++static int btf_sec_info_cmp(const void *a, const void *b) -++{ -++ const struct btf_sec_info *x = a; -++ const struct btf_sec_info *y = b; -++ -++ return (int)(x->off - y->off) ? : (int)(x->len - y->len); -++} -++ -++static int btf_check_sec_info(struct btf_verifier_env *env, -++ u32 btf_data_size) -++{ -++ struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)]; -++ u32 total, expected_total, i; -++ const struct btf_header *hdr; -++ const struct btf *btf; -++ -++ btf = env->btf; -++ hdr = &btf->hdr; -++ -++ /* Populate the secs from hdr */ -++ for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) -++ secs[i] = *(struct btf_sec_info *)((void *)hdr + -++ btf_sec_info_offset[i]); -++ -++ sort(secs, ARRAY_SIZE(btf_sec_info_offset), -++ sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL); -++ -++ /* Check for gaps and overlap among sections */ -++ total = 0; -++ expected_total = btf_data_size - hdr->hdr_len; -++ for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) { -++ if (expected_total < secs[i].off) { -++ btf_verifier_log(env, "Invalid section offset"); -++ return -EINVAL; -++ } -++ if (total < secs[i].off) { -++ /* gap */ -++ btf_verifier_log(env, "Unsupported section found"); -++ return -EINVAL; -++ } -++ if (total > secs[i].off) { -++ btf_verifier_log(env, "Section overlap found"); -++ return -EINVAL; -++ } -++ if (expected_total - total < secs[i].len) { -++ btf_verifier_log(env, -++ "Total section length too long"); -++ return -EINVAL; -++ } -++ total += secs[i].len; -++ } -++ -++ /* There is data other than hdr and known sections */ -++ if (expected_total != total) { -++ btf_verifier_log(env, "Unsupported section found"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static int btf_parse_hdr(struct btf_verifier_env *env) -++{ -++ u32 hdr_len, hdr_copy, btf_data_size; -++ const struct btf_header *hdr; -++ struct btf *btf; -++ int err; -++ -++ btf = env->btf; -++ btf_data_size = btf->data_size; -++ -++ if (btf_data_size < -++ offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) { -++ btf_verifier_log(env, "hdr_len not found"); -++ return -EINVAL; -++ } -++ -++ hdr = btf->data; -++ hdr_len = hdr->hdr_len; -++ if (btf_data_size < hdr_len) { -++ btf_verifier_log(env, "btf_header not found"); -++ return -EINVAL; -++ } -++ -++ /* Ensure the unsupported header fields are zero */ -++ if (hdr_len > sizeof(btf->hdr)) { -++ u8 *expected_zero = btf->data + sizeof(btf->hdr); -++ u8 *end = btf->data + hdr_len; -++ -++ for (; expected_zero < end; expected_zero++) { -++ if (*expected_zero) { -++ btf_verifier_log(env, "Unsupported btf_header"); -++ return -E2BIG; -++ } -++ } -++ } -++ -++ hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); -++ memcpy(&btf->hdr, btf->data, hdr_copy); -++ -++ hdr = &btf->hdr; -++ -++ btf_verifier_log_hdr(env, btf_data_size); -++ -++ if (hdr->magic != BTF_MAGIC) { -++ btf_verifier_log(env, "Invalid magic"); -++ return -EINVAL; -++ } -++ -++ if (hdr->version != BTF_VERSION) { -++ btf_verifier_log(env, "Unsupported version"); -++ return -ENOTSUPP; -++ } -++ -++ if (hdr->flags) { -++ btf_verifier_log(env, "Unsupported flags"); -++ return -ENOTSUPP; -++ } -++ -++ if (btf_data_size == hdr->hdr_len) { -++ btf_verifier_log(env, "No data"); -++ return -EINVAL; -++ } -++ -++ err = btf_check_sec_info(env, btf_data_size); -++ if (err) -++ return err; -++ -++ return 0; -++} -++ -++static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size, -++ u32 log_level, char __user *log_ubuf, u32 log_size) -++{ -++ struct btf_verifier_env *env = NULL; -++ struct bpf_verifier_log *log; -++ struct btf *btf = NULL; -++ u8 *data; -++ int err; -++ -++ if (btf_data_size > BTF_MAX_SIZE) -++ return ERR_PTR(-E2BIG); -++ -++ env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN); -++ if (!env) -++ return ERR_PTR(-ENOMEM); -++ -++ log = &env->log; -++ if (log_level || log_ubuf || log_size) { -++ /* user requested verbose verifier output -++ * and supplied buffer to store the verification trace -++ */ -++ log->level = log_level; -++ log->ubuf = log_ubuf; -++ log->len_total = log_size; -++ -++ /* log attributes have to be sane */ -++ if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 || -++ !log->level || !log->ubuf) { -++ err = -EINVAL; -++ goto errout; -++ } -++ } -++ -++ btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); -++ if (!btf) { -++ err = -ENOMEM; -++ goto errout; -++ } -++ env->btf = btf; -++ -++ data = kmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN); -++ if (!data) { -++ err = -ENOMEM; -++ goto errout; -++ } -++ -++ btf->data = data; -++ btf->data_size = btf_data_size; -++ -++ if (copy_from_user(data, btf_data, btf_data_size)) { -++ err = -EFAULT; -++ goto errout; -++ } -++ -++ err = btf_parse_hdr(env); -++ if (err) -++ goto errout; -++ -++ btf->nohdr_data = btf->data + btf->hdr.hdr_len; -++ -++ err = btf_parse_str_sec(env); -++ if (err) -++ goto errout; -++ -++ err = btf_parse_type_sec(env); -++ if (err) -++ goto errout; -++ -++ if (log->level && bpf_verifier_log_full(log)) { -++ err = -ENOSPC; -++ goto errout; -++ } -++ -++ btf_verifier_env_free(env); -++ refcount_set(&btf->refcnt, 1); -++ return btf; -++ -++errout: -++ btf_verifier_env_free(env); -++ if (btf) -++ btf_free(btf); -++ return ERR_PTR(err); -++} -++ -++void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, -++ struct seq_file *m) -++{ -++ const struct btf_type *t = btf_type_by_id(btf, type_id); -++ -++ btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m); -++} -++ -++#ifdef CONFIG_PROC_FS -++static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) -++{ -++ const struct btf *btf = filp->private_data; -++ -++ seq_printf(m, "btf_id:\t%u\n", btf->id); -++} -++#endif -++ -++static int btf_release(struct inode *inode, struct file *filp) -++{ -++ btf_put(filp->private_data); -++ return 0; -++} -++ -++const struct file_operations btf_fops = { -++#ifdef CONFIG_PROC_FS -++ .show_fdinfo = bpf_btf_show_fdinfo, -++#endif -++ .release = btf_release, -++}; -++ -++static int __btf_new_fd(struct btf *btf) -++{ -++ return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); -++} -++ -++int btf_new_fd(const union bpf_attr *attr) -++{ -++ struct btf *btf; -++ int ret; -++ -++ btf = btf_parse(u64_to_user_ptr(attr->btf), -++ attr->btf_size, attr->btf_log_level, -++ u64_to_user_ptr(attr->btf_log_buf), -++ attr->btf_log_size); -++ if (IS_ERR(btf)) -++ return PTR_ERR(btf); -++ -++ ret = btf_alloc_id(btf); -++ if (ret) { -++ btf_free(btf); -++ return ret; -++ } -++ -++ /* -++ * The BTF ID is published to the userspace. -++ * All BTF free must go through call_rcu() from -++ * now on (i.e. free by calling btf_put()). -++ */ -++ -++ ret = __btf_new_fd(btf); -++ if (ret < 0) -++ btf_put(btf); -++ -++ return ret; -++} -++ -++struct btf *btf_get_by_fd(int fd) -++{ -++ struct btf *btf; -++ struct fd f; -++ -++ f = fdget(fd); -++ -++ if (!f.file) -++ return ERR_PTR(-EBADF); -++ -++ if (f.file->f_op != &btf_fops) { -++ fdput(f); -++ return ERR_PTR(-EINVAL); -++ } -++ -++ btf = f.file->private_data; -++ refcount_inc(&btf->refcnt); -++ fdput(f); -++ -++ return btf; -++} -++ -++int btf_get_info_by_fd(const struct btf *btf, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct bpf_btf_info __user *uinfo; -++ struct bpf_btf_info info; -++ u32 info_copy, btf_copy; -++ void __user *ubtf; -++ u32 uinfo_len; -++ -++ uinfo = u64_to_user_ptr(attr->info.info); -++ uinfo_len = attr->info.info_len; -++ -++ info_copy = min_t(u32, uinfo_len, sizeof(info)); -++ memset(&info, 0, sizeof(info)); -++ if (copy_from_user(&info, uinfo, info_copy)) -++ return -EFAULT; -++ -++ info.id = btf->id; -++ ubtf = u64_to_user_ptr(info.btf); -++ btf_copy = min_t(u32, btf->data_size, info.btf_size); -++ if (copy_to_user(ubtf, btf->data, btf_copy)) -++ return -EFAULT; -++ info.btf_size = btf->data_size; -++ -++ if (copy_to_user(uinfo, &info, info_copy) || -++ put_user(info_copy, &uattr->info.info_len)) -++ return -EFAULT; -++ -++ return 0; -++} -++ -++int btf_get_fd_by_id(u32 id) -++{ -++ struct btf *btf; -++ int fd; -++ -++ rcu_read_lock(); -++ btf = idr_find(&btf_idr, id); -++ if (!btf || !refcount_inc_not_zero(&btf->refcnt)) -++ btf = ERR_PTR(-ENOENT); -++ rcu_read_unlock(); -++ -++ if (IS_ERR(btf)) -++ return PTR_ERR(btf); -++ -++ fd = __btf_new_fd(btf); -++ if (fd < 0) -++ btf_put(btf); -++ -++ return fd; -++} -++ -++u32 btf_id(const struct btf *btf) -++{ -++ return btf->id; -++} -+--- /dev/null -++++ b/kernel/bpf/cgroup.c -+@@ -0,0 +1,1581 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* -++ * Functions to manage eBPF programs attached to cgroups -++ * -++ * Copyright (c) 2016 Daniel Mack -++ */ -++ -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++#include "../cgroup/cgroup-internal.h" -++ -++DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key); -++EXPORT_SYMBOL(cgroup_bpf_enabled_key); -++ -++void cgroup_bpf_offline(struct cgroup *cgrp) -++{ -++ cgroup_get(cgrp); -++ percpu_ref_kill(&cgrp->bpf.refcnt); -++} -++ -++/** -++ * cgroup_bpf_release() - put references of all bpf programs and -++ * release all cgroup bpf data -++ * @work: work structure embedded into the cgroup to modify -++ */ -++static void cgroup_bpf_release(struct work_struct *work) -++{ -++ struct cgroup *p, *cgrp = container_of(work, struct cgroup, -++ bpf.release_work); -++ enum bpf_cgroup_storage_type stype; -++ struct bpf_prog_array *old_array; -++ unsigned int type; -++ -++ mutex_lock(&cgroup_mutex); -++ -++ for (type = 0; type < ARRAY_SIZE(cgrp->bpf.progs); type++) { -++ struct list_head *progs = &cgrp->bpf.progs[type]; -++ struct bpf_prog_list *pl, *tmp; -++ -++ list_for_each_entry_safe(pl, tmp, progs, node) { -++ list_del(&pl->node); -++ bpf_prog_put(pl->prog); -++ for_each_cgroup_storage_type(stype) { -++ bpf_cgroup_storage_unlink(pl->storage[stype]); -++ bpf_cgroup_storage_free(pl->storage[stype]); -++ } -++ kfree(pl); -++ static_branch_dec(&cgroup_bpf_enabled_key); -++ } -++ old_array = rcu_dereference_protected( -++ cgrp->bpf.effective[type], -++ lockdep_is_held(&cgroup_mutex)); -++ bpf_prog_array_free(old_array); -++ } -++ -++ mutex_unlock(&cgroup_mutex); -++ -++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) -++ cgroup_bpf_put(p); -++ -++ percpu_ref_exit(&cgrp->bpf.refcnt); -++ cgroup_put(cgrp); -++} -++ -++/** -++ * cgroup_bpf_release_fn() - callback used to schedule releasing -++ * of bpf cgroup data -++ * @ref: percpu ref counter structure -++ */ -++static void cgroup_bpf_release_fn(struct percpu_ref *ref) -++{ -++ struct cgroup *cgrp = container_of(ref, struct cgroup, bpf.refcnt); -++ -++ INIT_WORK(&cgrp->bpf.release_work, cgroup_bpf_release); -++ queue_work(system_wq, &cgrp->bpf.release_work); -++} -++ -++/* count number of elements in the list. -++ * it's slow but the list cannot be long -++ */ -++static u32 prog_list_length(struct list_head *head) -++{ -++ struct bpf_prog_list *pl; -++ u32 cnt = 0; -++ -++ list_for_each_entry(pl, head, node) { -++ if (!pl->prog) -++ continue; -++ cnt++; -++ } -++ return cnt; -++} -++ -++/* if parent has non-overridable prog attached, -++ * disallow attaching new programs to the descendent cgroup. -++ * if parent has overridable or multi-prog, allow attaching -++ */ -++static bool hierarchy_allows_attach(struct cgroup *cgrp, -++ enum bpf_attach_type type, -++ u32 new_flags) -++{ -++ struct cgroup *p; -++ -++ p = cgroup_parent(cgrp); -++ if (!p) -++ return true; -++ do { -++ u32 flags = p->bpf.flags[type]; -++ u32 cnt; -++ -++ if (flags & BPF_F_ALLOW_MULTI) -++ return true; -++ cnt = prog_list_length(&p->bpf.progs[type]); -++ WARN_ON_ONCE(cnt > 1); -++ if (cnt == 1) -++ return !!(flags & BPF_F_ALLOW_OVERRIDE); -++ p = cgroup_parent(p); -++ } while (p); -++ return true; -++} -++ -++/* compute a chain of effective programs for a given cgroup: -++ * start from the list of programs in this cgroup and add -++ * all parent programs. -++ * Note that parent's F_ALLOW_OVERRIDE-type program is yielding -++ * to programs in this cgroup -++ */ -++static int compute_effective_progs(struct cgroup *cgrp, -++ enum bpf_attach_type type, -++ struct bpf_prog_array **array) -++{ -++ enum bpf_cgroup_storage_type stype; -++ struct bpf_prog_array *progs; -++ struct bpf_prog_list *pl; -++ struct cgroup *p = cgrp; -++ int cnt = 0; -++ -++ /* count number of effective programs by walking parents */ -++ do { -++ if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) -++ cnt += prog_list_length(&p->bpf.progs[type]); -++ p = cgroup_parent(p); -++ } while (p); -++ -++ progs = bpf_prog_array_alloc(cnt, GFP_KERNEL); -++ if (!progs) -++ return -ENOMEM; -++ -++ /* populate the array with effective progs */ -++ cnt = 0; -++ p = cgrp; -++ do { -++ if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI)) -++ continue; -++ -++ list_for_each_entry(pl, &p->bpf.progs[type], node) { -++ if (!pl->prog) -++ continue; -++ -++ progs->items[cnt].prog = pl->prog; -++ for_each_cgroup_storage_type(stype) -++ progs->items[cnt].cgroup_storage[stype] = -++ pl->storage[stype]; -++ cnt++; -++ } -++ } while ((p = cgroup_parent(p))); -++ -++ *array = progs; -++ return 0; -++} -++ -++static void activate_effective_progs(struct cgroup *cgrp, -++ enum bpf_attach_type type, -++ struct bpf_prog_array *old_array) -++{ -++ rcu_swap_protected(cgrp->bpf.effective[type], old_array, -++ lockdep_is_held(&cgroup_mutex)); -++ /* free prog array after grace period, since __cgroup_bpf_run_*() -++ * might be still walking the array -++ */ -++ bpf_prog_array_free(old_array); -++} -++ -++/** -++ * cgroup_bpf_inherit() - inherit effective programs from parent -++ * @cgrp: the cgroup to modify -++ */ -++int cgroup_bpf_inherit(struct cgroup *cgrp) -++{ -++/* has to use marco instead of const int, since compiler thinks -++ * that array below is variable length -++ */ -++#define NR ARRAY_SIZE(cgrp->bpf.effective) -++ struct bpf_prog_array *arrays[NR] = {}; -++ struct cgroup *p; -++ int ret, i; -++ -++ ret = percpu_ref_init(&cgrp->bpf.refcnt, cgroup_bpf_release_fn, 0, -++ GFP_KERNEL); -++ if (ret) -++ return ret; -++ -++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) -++ cgroup_bpf_get(p); -++ -++ for (i = 0; i < NR; i++) -++ INIT_LIST_HEAD(&cgrp->bpf.progs[i]); -++ -++ for (i = 0; i < NR; i++) -++ if (compute_effective_progs(cgrp, i, &arrays[i])) -++ goto cleanup; -++ -++ for (i = 0; i < NR; i++) -++ activate_effective_progs(cgrp, i, arrays[i]); -++ -++ return 0; -++cleanup: -++ for (i = 0; i < NR; i++) -++ bpf_prog_array_free(arrays[i]); -++ -++ for (p = cgroup_parent(cgrp); p; p = cgroup_parent(p)) -++ cgroup_bpf_put(p); -++ -++ percpu_ref_exit(&cgrp->bpf.refcnt); -++ -++ return -ENOMEM; -++} -++ -++static int update_effective_progs(struct cgroup *cgrp, -++ enum bpf_attach_type type) -++{ -++ struct cgroup_subsys_state *css; -++ int err; -++ -++ /* allocate and recompute effective prog arrays */ -++ css_for_each_descendant_pre(css, &cgrp->self) { -++ struct cgroup *desc = container_of(css, struct cgroup, self); -++ -++ if (percpu_ref_is_zero(&desc->bpf.refcnt)) -++ continue; -++ -++ err = compute_effective_progs(desc, type, &desc->bpf.inactive); -++ if (err) -++ goto cleanup; -++ } -++ -++ /* all allocations were successful. Activate all prog arrays */ -++ css_for_each_descendant_pre(css, &cgrp->self) { -++ struct cgroup *desc = container_of(css, struct cgroup, self); -++ -++ if (percpu_ref_is_zero(&desc->bpf.refcnt)) { -++ if (unlikely(desc->bpf.inactive)) { -++ bpf_prog_array_free(desc->bpf.inactive); -++ desc->bpf.inactive = NULL; -++ } -++ continue; -++ } -++ -++ activate_effective_progs(desc, type, desc->bpf.inactive); -++ desc->bpf.inactive = NULL; -++ } -++ -++ return 0; -++ -++cleanup: -++ /* oom while computing effective. Free all computed effective arrays -++ * since they were not activated -++ */ -++ css_for_each_descendant_pre(css, &cgrp->self) { -++ struct cgroup *desc = container_of(css, struct cgroup, self); -++ -++ bpf_prog_array_free(desc->bpf.inactive); -++ desc->bpf.inactive = NULL; -++ } -++ -++ return err; -++} -++ -++#define BPF_CGROUP_MAX_PROGS 64 -++ -++/** -++ * __cgroup_bpf_attach() - Attach the program to a cgroup, and -++ * propagate the change to descendants -++ * @cgrp: The cgroup which descendants to traverse -++ * @prog: A program to attach -++ * @type: Type of attach operation -++ * @flags: Option flags -++ * -++ * Must be called with cgroup_mutex held. -++ */ -++int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type, u32 flags) -++{ -++ struct list_head *progs = &cgrp->bpf.progs[type]; -++ struct bpf_prog *old_prog = NULL; -++ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; -++ struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; -++ enum bpf_cgroup_storage_type stype; -++ struct bpf_prog_list *pl; -++ bool pl_was_allocated; -++ int err; -++ -++ if ((flags & BPF_F_ALLOW_OVERRIDE) && (flags & BPF_F_ALLOW_MULTI)) -++ /* invalid combination */ -++ return -EINVAL; -++ -++ if (!hierarchy_allows_attach(cgrp, type, flags)) -++ return -EPERM; -++ -++ if (!list_empty(progs) && cgrp->bpf.flags[type] != flags) -++ /* Disallow attaching non-overridable on top -++ * of existing overridable in this cgroup. -++ * Disallow attaching multi-prog if overridable or none -++ */ -++ return -EPERM; -++ -++ if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS) -++ return -E2BIG; -++ -++ for_each_cgroup_storage_type(stype) { -++ storage[stype] = bpf_cgroup_storage_alloc(prog, stype); -++ if (IS_ERR(storage[stype])) { -++ storage[stype] = NULL; -++ for_each_cgroup_storage_type(stype) -++ bpf_cgroup_storage_free(storage[stype]); -++ return -ENOMEM; -++ } -++ } -++ -++ if (flags & BPF_F_ALLOW_MULTI) { -++ list_for_each_entry(pl, progs, node) { -++ if (pl->prog == prog) { -++ /* disallow attaching the same prog twice */ -++ for_each_cgroup_storage_type(stype) -++ bpf_cgroup_storage_free(storage[stype]); -++ return -EINVAL; -++ } -++ } -++ -++ pl = kmalloc(sizeof(*pl), GFP_KERNEL); -++ if (!pl) { -++ for_each_cgroup_storage_type(stype) -++ bpf_cgroup_storage_free(storage[stype]); -++ return -ENOMEM; -++ } -++ -++ pl_was_allocated = true; -++ pl->prog = prog; -++ for_each_cgroup_storage_type(stype) -++ pl->storage[stype] = storage[stype]; -++ list_add_tail(&pl->node, progs); -++ } else { -++ if (list_empty(progs)) { -++ pl = kmalloc(sizeof(*pl), GFP_KERNEL); -++ if (!pl) { -++ for_each_cgroup_storage_type(stype) -++ bpf_cgroup_storage_free(storage[stype]); -++ return -ENOMEM; -++ } -++ pl_was_allocated = true; -++ list_add_tail(&pl->node, progs); -++ } else { -++ pl = list_first_entry(progs, typeof(*pl), node); -++ old_prog = pl->prog; -++ for_each_cgroup_storage_type(stype) { -++ old_storage[stype] = pl->storage[stype]; -++ bpf_cgroup_storage_unlink(old_storage[stype]); -++ } -++ pl_was_allocated = false; -++ } -++ pl->prog = prog; -++ for_each_cgroup_storage_type(stype) -++ pl->storage[stype] = storage[stype]; -++ } -++ -++ cgrp->bpf.flags[type] = flags; -++ -++ err = update_effective_progs(cgrp, type); -++ if (err) -++ goto cleanup; -++ -++ static_branch_inc(&cgroup_bpf_enabled_key); -++ for_each_cgroup_storage_type(stype) { -++ if (!old_storage[stype]) -++ continue; -++ bpf_cgroup_storage_free(old_storage[stype]); -++ } -++ if (old_prog) { -++ bpf_prog_put(old_prog); -++ static_branch_dec(&cgroup_bpf_enabled_key); -++ } -++ for_each_cgroup_storage_type(stype) -++ bpf_cgroup_storage_link(storage[stype], cgrp, type); -++ return 0; -++ -++cleanup: -++ /* and cleanup the prog list */ -++ pl->prog = old_prog; -++ for_each_cgroup_storage_type(stype) { -++ bpf_cgroup_storage_free(pl->storage[stype]); -++ pl->storage[stype] = old_storage[stype]; -++ bpf_cgroup_storage_link(old_storage[stype], cgrp, type); -++ } -++ if (pl_was_allocated) { -++ list_del(&pl->node); -++ kfree(pl); -++ } -++ return err; -++} -++ -++/** -++ * __cgroup_bpf_detach() - Detach the program from a cgroup, and -++ * propagate the change to descendants -++ * @cgrp: The cgroup which descendants to traverse -++ * @prog: A program to detach or NULL -++ * @type: Type of detach operation -++ * -++ * Must be called with cgroup_mutex held. -++ */ -++int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, -++ enum bpf_attach_type type) -++{ -++ struct list_head *progs = &cgrp->bpf.progs[type]; -++ enum bpf_cgroup_storage_type stype; -++ u32 flags = cgrp->bpf.flags[type]; -++ struct bpf_prog *old_prog = NULL; -++ struct bpf_prog_list *pl; -++ int err; -++ -++ if (flags & BPF_F_ALLOW_MULTI) { -++ if (!prog) -++ /* to detach MULTI prog the user has to specify valid FD -++ * of the program to be detached -++ */ -++ return -EINVAL; -++ } else { -++ if (list_empty(progs)) -++ /* report error when trying to detach and nothing is attached */ -++ return -ENOENT; -++ } -++ -++ if (flags & BPF_F_ALLOW_MULTI) { -++ /* find the prog and detach it */ -++ list_for_each_entry(pl, progs, node) { -++ if (pl->prog != prog) -++ continue; -++ old_prog = prog; -++ /* mark it deleted, so it's ignored while -++ * recomputing effective -++ */ -++ pl->prog = NULL; -++ break; -++ } -++ if (!old_prog) -++ return -ENOENT; -++ } else { -++ /* to maintain backward compatibility NONE and OVERRIDE cgroups -++ * allow detaching with invalid FD (prog==NULL) -++ */ -++ pl = list_first_entry(progs, typeof(*pl), node); -++ old_prog = pl->prog; -++ pl->prog = NULL; -++ } -++ -++ err = update_effective_progs(cgrp, type); -++ if (err) -++ goto cleanup; -++ -++ /* now can actually delete it from this cgroup list */ -++ list_del(&pl->node); -++ for_each_cgroup_storage_type(stype) { -++ bpf_cgroup_storage_unlink(pl->storage[stype]); -++ bpf_cgroup_storage_free(pl->storage[stype]); -++ } -++ kfree(pl); -++ if (list_empty(progs)) -++ /* last program was detached, reset flags to zero */ -++ cgrp->bpf.flags[type] = 0; -++ -++ bpf_prog_put(old_prog); -++ static_branch_dec(&cgroup_bpf_enabled_key); -++ return 0; -++ -++cleanup: -++ /* and restore back old_prog */ -++ pl->prog = old_prog; -++ return err; -++} -++ -++/* Must be called with cgroup_mutex held to avoid races. */ -++int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); -++ enum bpf_attach_type type = attr->query.attach_type; -++ struct list_head *progs = &cgrp->bpf.progs[type]; -++ u32 flags = cgrp->bpf.flags[type]; -++ struct bpf_prog_array *effective; -++ int cnt, ret = 0, i; -++ -++ effective = rcu_dereference_protected(cgrp->bpf.effective[type], -++ lockdep_is_held(&cgroup_mutex)); -++ -++ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) -++ cnt = bpf_prog_array_length(effective); -++ else -++ cnt = prog_list_length(progs); -++ -++ if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) -++ return -EFAULT; -++ if (copy_to_user(&uattr->query.prog_cnt, &cnt, sizeof(cnt))) -++ return -EFAULT; -++ if (attr->query.prog_cnt == 0 || !prog_ids || !cnt) -++ /* return early if user requested only program count + flags */ -++ return 0; -++ if (attr->query.prog_cnt < cnt) { -++ cnt = attr->query.prog_cnt; -++ ret = -ENOSPC; -++ } -++ -++ if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) { -++ return bpf_prog_array_copy_to_user(effective, prog_ids, cnt); -++ } else { -++ struct bpf_prog_list *pl; -++ u32 id; -++ -++ i = 0; -++ list_for_each_entry(pl, progs, node) { -++ id = pl->prog->aux->id; -++ if (copy_to_user(prog_ids + i, &id, sizeof(id))) -++ return -EFAULT; -++ if (++i == cnt) -++ break; -++ } -++ } -++ return ret; -++} -++ -++int cgroup_bpf_prog_attach(const union bpf_attr *attr, -++ enum bpf_prog_type ptype, struct bpf_prog *prog) -++{ -++ struct cgroup *cgrp; -++ int ret; -++ -++ cgrp = cgroup_get_from_fd(attr->target_fd); -++ if (IS_ERR(cgrp)) -++ return PTR_ERR(cgrp); -++ -++ ret = cgroup_bpf_attach(cgrp, prog, attr->attach_type, -++ attr->attach_flags); -++ cgroup_put(cgrp); -++ return ret; -++} -++ -++int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype) -++{ -++ struct bpf_prog *prog; -++ struct cgroup *cgrp; -++ int ret; -++ -++ cgrp = cgroup_get_from_fd(attr->target_fd); -++ if (IS_ERR(cgrp)) -++ return PTR_ERR(cgrp); -++ -++ prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype); -++ if (IS_ERR(prog)) -++ prog = NULL; -++ -++ ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, 0); -++ if (prog) -++ bpf_prog_put(prog); -++ -++ cgroup_put(cgrp); -++ return ret; -++} -++ -++int cgroup_bpf_prog_query(const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct cgroup *cgrp; -++ int ret; -++ -++ cgrp = cgroup_get_from_fd(attr->query.target_fd); -++ if (IS_ERR(cgrp)) -++ return PTR_ERR(cgrp); -++ -++ ret = cgroup_bpf_query(cgrp, attr, uattr); -++ -++ cgroup_put(cgrp); -++ return ret; -++} -++ -++/** -++ * __cgroup_bpf_run_filter_skb() - Run a program for packet filtering -++ * @sk: The socket sending or receiving traffic -++ * @skb: The skb that is being sent or received -++ * @type: The type of program to be exectuted -++ * -++ * If no socket is passed, or the socket is not of type INET or INET6, -++ * this function does nothing and returns 0. -++ * -++ * The program type passed in via @type must be suitable for network -++ * filtering. No further check is performed to assert that. -++ * -++ * For egress packets, this function can return: -++ * NET_XMIT_SUCCESS (0) - continue with packet output -++ * NET_XMIT_DROP (1) - drop packet and notify TCP to call cwr -++ * NET_XMIT_CN (2) - continue with packet output and notify TCP -++ * to call cwr -++ * -EPERM - drop packet -++ * -++ * For ingress packets, this function will return -EPERM if any -++ * attached program was found and if it returned != 1 during execution. -++ * Otherwise 0 is returned. -++ */ -++int __cgroup_bpf_run_filter_skb(struct sock *sk, -++ struct sk_buff *skb, -++ enum bpf_attach_type type) -++{ -++ unsigned int offset = skb->data - skb_network_header(skb); -++ struct sock *save_sk; -++ void *saved_data_end; -++ struct cgroup *cgrp; -++ int ret; -++ -++ if (!sk || !sk_fullsock(sk)) -++ return 0; -++ -++ if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6) -++ return 0; -++ -++ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ save_sk = skb->sk; -++ skb->sk = sk; -++ __skb_push(skb, offset); -++ -++ /* compute pointers for the bpf prog */ -++ bpf_compute_and_save_data_end(skb, &saved_data_end); -++ -++ if (type == BPF_CGROUP_INET_EGRESS) { -++ ret = BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY( -++ cgrp->bpf.effective[type], skb, __bpf_prog_run_save_cb); -++ } else { -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], skb, -++ __bpf_prog_run_save_cb); -++ ret = (ret == 1 ? 0 : -EPERM); -++ } -++ bpf_restore_data_end(skb, saved_data_end); -++ __skb_pull(skb, offset); -++ skb->sk = save_sk; -++ -++ return ret; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_skb); -++ -++/** -++ * __cgroup_bpf_run_filter_sk() - Run a program on a sock -++ * @sk: sock structure to manipulate -++ * @type: The type of program to be exectuted -++ * -++ * socket is passed is expected to be of type INET or INET6. -++ * -++ * The program type passed in via @type must be suitable for sock -++ * filtering. No further check is performed to assert that. -++ * -++ * This function will return %-EPERM if any if an attached program was found -++ * and if it returned != 1 during execution. In all other cases, 0 is returned. -++ */ -++int __cgroup_bpf_run_filter_sk(struct sock *sk, -++ enum bpf_attach_type type) -++{ -++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ int ret; -++ -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sk, BPF_PROG_RUN); -++ return ret == 1 ? 0 : -EPERM; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); -++ -++/** -++ * __cgroup_bpf_run_filter_sock_addr() - Run a program on a sock and -++ * provided by user sockaddr -++ * @sk: sock struct that will use sockaddr -++ * @uaddr: sockaddr struct provided by user -++ * @type: The type of program to be exectuted -++ * @t_ctx: Pointer to attach type specific context -++ * -++ * socket is expected to be of type INET or INET6. -++ * -++ * This function will return %-EPERM if an attached program is found and -++ * returned value != 1 during execution. In all other cases, 0 is returned. -++ */ -++int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, -++ struct sockaddr *uaddr, -++ enum bpf_attach_type type, -++ void *t_ctx) -++{ -++ struct bpf_sock_addr_kern ctx = { -++ .sk = sk, -++ .uaddr = uaddr, -++ .t_ctx = t_ctx, -++ }; -++ struct sockaddr_storage unspec; -++ struct cgroup *cgrp; -++ int ret; -++ -++ /* Check socket family since not all sockets represent network -++ * endpoint (e.g. AF_UNIX). -++ */ -++ if (sk->sk_family != AF_INET && sk->sk_family != AF_INET6) -++ return 0; -++ -++ if (!ctx.uaddr) { -++ memset(&unspec, 0, sizeof(unspec)); -++ ctx.uaddr = (struct sockaddr *)&unspec; -++ } -++ -++ cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN); -++ -++ return ret == 1 ? 0 : -EPERM; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr); -++ -++/** -++ * __cgroup_bpf_run_filter_sock_ops() - Run a program on a sock -++ * @sk: socket to get cgroup from -++ * @sock_ops: bpf_sock_ops_kern struct to pass to program. Contains -++ * sk with connection information (IP addresses, etc.) May not contain -++ * cgroup info if it is a req sock. -++ * @type: The type of program to be exectuted -++ * -++ * socket passed is expected to be of type INET or INET6. -++ * -++ * The program type passed in via @type must be suitable for sock_ops -++ * filtering. No further check is performed to assert that. -++ * -++ * This function will return %-EPERM if any if an attached program was found -++ * and if it returned != 1 during execution. In all other cases, 0 is returned. -++ */ -++int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, -++ struct bpf_sock_ops_kern *sock_ops, -++ enum bpf_attach_type type) -++{ -++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ int ret; -++ -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], sock_ops, -++ BPF_PROG_RUN); -++ return ret == 1 ? 0 : -EPERM; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops); -++ -++int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, -++ short access, enum bpf_attach_type type) -++{ -++ struct cgroup *cgrp; -++ struct bpf_cgroup_dev_ctx ctx = { -++ .access_type = (access << 16) | dev_type, -++ .major = major, -++ .minor = minor, -++ }; -++ int allow = 1; -++ -++ rcu_read_lock(); -++ cgrp = task_dfl_cgroup(current); -++ allow = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, -++ BPF_PROG_RUN); -++ rcu_read_unlock(); -++ -++ return !allow; -++} -++EXPORT_SYMBOL(__cgroup_bpf_check_dev_permission); -++ -++static const struct bpf_func_proto * -++cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ case BPF_FUNC_map_lookup_elem: -++ return &bpf_map_lookup_elem_proto; -++ case BPF_FUNC_map_update_elem: -++ return &bpf_map_update_elem_proto; -++ case BPF_FUNC_map_delete_elem: -++ return &bpf_map_delete_elem_proto; -++ case BPF_FUNC_map_push_elem: -++ return &bpf_map_push_elem_proto; -++ case BPF_FUNC_map_pop_elem: -++ return &bpf_map_pop_elem_proto; -++ case BPF_FUNC_map_peek_elem: -++ return &bpf_map_peek_elem_proto; -++ case BPF_FUNC_get_current_uid_gid: -++ return &bpf_get_current_uid_gid_proto; -++ case BPF_FUNC_get_local_storage: -++ return &bpf_get_local_storage_proto; -++ case BPF_FUNC_get_current_cgroup_id: -++ return &bpf_get_current_cgroup_id_proto; -++ case BPF_FUNC_trace_printk: -++ if (capable(CAP_SYS_ADMIN)) -++ return bpf_get_trace_printk_proto(); -++ /* fall through */ -++ default: -++ return NULL; -++ } -++} -++ -++static const struct bpf_func_proto * -++cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ return cgroup_base_func_proto(func_id, prog); -++} -++ -++static bool cgroup_dev_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ const int size_default = sizeof(__u32); -++ -++ if (type == BPF_WRITE) -++ return false; -++ -++ if (off < 0 || off + size > sizeof(struct bpf_cgroup_dev_ctx)) -++ return false; -++ /* The verifier guarantees that size > 0. */ -++ if (off % size != 0) -++ return false; -++ -++ switch (off) { -++ case bpf_ctx_range(struct bpf_cgroup_dev_ctx, access_type): -++ bpf_ctx_record_field_size(info, size_default); -++ if (!bpf_ctx_narrow_access_ok(off, size, size_default)) -++ return false; -++ break; -++ default: -++ if (size != size_default) -++ return false; -++ } -++ -++ return true; -++} -++ -++const struct bpf_prog_ops cg_dev_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops cg_dev_verifier_ops = { -++ .get_func_proto = cgroup_dev_func_proto, -++ .is_valid_access = cgroup_dev_is_valid_access, -++}; -++ -++/** -++ * __cgroup_bpf_run_filter_sysctl - Run a program on sysctl -++ * -++ * @head: sysctl table header -++ * @table: sysctl table -++ * @write: sysctl is being read (= 0) or written (= 1) -++ * @buf: pointer to buffer passed by user space -++ * @pcount: value-result argument: value is size of buffer pointed to by @buf, -++ * result is size of @new_buf if program set new value, initial value -++ * otherwise -++ * @ppos: value-result argument: value is position at which read from or write -++ * to sysctl is happening, result is new position if program overrode it, -++ * initial value otherwise -++ * @new_buf: pointer to pointer to new buffer that will be allocated if program -++ * overrides new value provided by user space on sysctl write -++ * NOTE: it's caller responsibility to free *new_buf if it was set -++ * @type: type of program to be executed -++ * -++ * Program is run when sysctl is being accessed, either read or written, and -++ * can allow or deny such access. -++ * -++ * This function will return %-EPERM if an attached program is found and -++ * returned value != 1 during execution. In all other cases 0 is returned. -++ */ -++int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, -++ struct ctl_table *table, int write, -++ void __user *buf, size_t *pcount, -++ loff_t *ppos, void **new_buf, -++ enum bpf_attach_type type) -++{ -++ struct bpf_sysctl_kern ctx = { -++ .head = head, -++ .table = table, -++ .write = write, -++ .ppos = ppos, -++ .cur_val = NULL, -++ .cur_len = PAGE_SIZE, -++ .new_val = NULL, -++ .new_len = 0, -++ .new_updated = 0, -++ }; -++ struct cgroup *cgrp; -++ int ret; -++ -++ ctx.cur_val = kmalloc_track_caller(ctx.cur_len, GFP_KERNEL); -++ if (ctx.cur_val) { -++ mm_segment_t old_fs; -++ loff_t pos = 0; -++ -++ old_fs = get_fs(); -++ set_fs(KERNEL_DS); -++ if (table->proc_handler(table, 0, (void __user *)ctx.cur_val, -++ &ctx.cur_len, &pos)) { -++ /* Let BPF program decide how to proceed. */ -++ ctx.cur_len = 0; -++ } -++ set_fs(old_fs); -++ } else { -++ /* Let BPF program decide how to proceed. */ -++ ctx.cur_len = 0; -++ } -++ -++ if (write && buf && *pcount) { -++ /* BPF program should be able to override new value with a -++ * buffer bigger than provided by user. -++ */ -++ ctx.new_val = kmalloc_track_caller(PAGE_SIZE, GFP_KERNEL); -++ ctx.new_len = min_t(size_t, PAGE_SIZE, *pcount); -++ if (!ctx.new_val || -++ copy_from_user(ctx.new_val, buf, ctx.new_len)) -++ /* Let BPF program decide how to proceed. */ -++ ctx.new_len = 0; -++ } -++ -++ rcu_read_lock(); -++ cgrp = task_dfl_cgroup(current); -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN); -++ rcu_read_unlock(); -++ -++ kfree(ctx.cur_val); -++ -++ if (ret == 1 && ctx.new_updated) { -++ *new_buf = ctx.new_val; -++ *pcount = ctx.new_len; -++ } else { -++ kfree(ctx.new_val); -++ } -++ -++ return ret == 1 ? 0 : -EPERM; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_sysctl); -++ -++#ifdef CONFIG_NET -++static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, -++ enum bpf_attach_type attach_type) -++{ -++ struct bpf_prog_array *prog_array; -++ bool empty; -++ -++ rcu_read_lock(); -++ prog_array = rcu_dereference(cgrp->bpf.effective[attach_type]); -++ empty = bpf_prog_array_is_empty(prog_array); -++ rcu_read_unlock(); -++ -++ return empty; -++} -++ -++static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) -++{ -++ if (unlikely(max_optlen < 0)) -++ return -EINVAL; -++ -++ if (unlikely(max_optlen > PAGE_SIZE)) { -++ /* We don't expose optvals that are greater than PAGE_SIZE -++ * to the BPF program. -++ */ -++ max_optlen = PAGE_SIZE; -++ } -++ -++ ctx->optval = kzalloc(max_optlen, GFP_USER); -++ if (!ctx->optval) -++ return -ENOMEM; -++ -++ ctx->optval_end = ctx->optval + max_optlen; -++ -++ return max_optlen; -++} -++ -++static void sockopt_free_buf(struct bpf_sockopt_kern *ctx) -++{ -++ kfree(ctx->optval); -++} -++ -++int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, -++ int *optname, char __user *optval, -++ int *optlen, char **kernel_optval) -++{ -++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ struct bpf_sockopt_kern ctx = { -++ .sk = sk, -++ .level = *level, -++ .optname = *optname, -++ }; -++ int ret, max_optlen; -++ -++ /* Opportunistic check to see whether we have any BPF program -++ * attached to the hook so we don't waste time allocating -++ * memory and locking the socket. -++ */ -++ if (!cgroup_bpf_enabled || -++ __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT)) -++ return 0; -++ -++ /* Allocate a bit more than the initial user buffer for -++ * BPF program. The canonical use case is overriding -++ * TCP_CONGESTION(nv) to TCP_CONGESTION(cubic). -++ */ -++ max_optlen = max_t(int, 16, *optlen); -++ -++ max_optlen = sockopt_alloc_buf(&ctx, max_optlen); -++ if (max_optlen < 0) -++ return max_optlen; -++ -++ ctx.optlen = *optlen; -++ -++ if (copy_from_user(ctx.optval, optval, min(*optlen, max_optlen)) != 0) { -++ ret = -EFAULT; -++ goto out; -++ } -++ -++ lock_sock(sk); -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_SETSOCKOPT], -++ &ctx, BPF_PROG_RUN); -++ release_sock(sk); -++ -++ if (!ret) { -++ ret = -EPERM; -++ goto out; -++ } -++ -++ if (ctx.optlen == -1) { -++ /* optlen set to -1, bypass kernel */ -++ ret = 1; -++ } else if (ctx.optlen > max_optlen || ctx.optlen < -1) { -++ /* optlen is out of bounds */ -++ ret = -EFAULT; -++ } else { -++ /* optlen within bounds, run kernel handler */ -++ ret = 0; -++ -++ /* export any potential modifications */ -++ *level = ctx.level; -++ *optname = ctx.optname; -++ -++ /* optlen == 0 from BPF indicates that we should -++ * use original userspace data. -++ */ -++ if (ctx.optlen != 0) { -++ *optlen = ctx.optlen; -++ *kernel_optval = ctx.optval; -++ /* export and don't free sockopt buf */ -++ return 0; -++ } -++ } -++ -++out: -++ sockopt_free_buf(&ctx); -++ return ret; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_setsockopt); -++ -++int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, -++ int optname, char __user *optval, -++ int __user *optlen, int max_optlen, -++ int retval) -++{ -++ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); -++ struct bpf_sockopt_kern ctx = { -++ .sk = sk, -++ .level = level, -++ .optname = optname, -++ .retval = retval, -++ }; -++ int ret; -++ -++ /* Opportunistic check to see whether we have any BPF program -++ * attached to the hook so we don't waste time allocating -++ * memory and locking the socket. -++ */ -++ if (!cgroup_bpf_enabled || -++ __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT)) -++ return retval; -++ -++ ctx.optlen = max_optlen; -++ -++ max_optlen = sockopt_alloc_buf(&ctx, max_optlen); -++ if (max_optlen < 0) -++ return max_optlen; -++ -++ if (!retval) { -++ /* If kernel getsockopt finished successfully, -++ * copy whatever was returned to the user back -++ * into our temporary buffer. Set optlen to the -++ * one that kernel returned as well to let -++ * BPF programs inspect the value. -++ */ -++ -++ if (get_user(ctx.optlen, optlen)) { -++ ret = -EFAULT; -++ goto out; -++ } -++ -++ if (ctx.optlen < 0) { -++ ret = -EFAULT; -++ goto out; -++ } -++ -++ if (copy_from_user(ctx.optval, optval, -++ min(ctx.optlen, max_optlen)) != 0) { -++ ret = -EFAULT; -++ goto out; -++ } -++ } -++ -++ lock_sock(sk); -++ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT], -++ &ctx, BPF_PROG_RUN); -++ release_sock(sk); -++ -++ if (!ret) { -++ ret = -EPERM; -++ goto out; -++ } -++ -++ if (ctx.optlen > max_optlen || ctx.optlen < 0) { -++ ret = -EFAULT; -++ goto out; -++ } -++ -++ /* BPF programs only allowed to set retval to 0, not some -++ * arbitrary value. -++ */ -++ if (ctx.retval != 0 && ctx.retval != retval) { -++ ret = -EFAULT; -++ goto out; -++ } -++ -++ if (ctx.optlen != 0) { -++ if (copy_to_user(optval, ctx.optval, ctx.optlen) || -++ put_user(ctx.optlen, optlen)) { -++ ret = -EFAULT; -++ goto out; -++ } -++ } -++ -++ ret = ctx.retval; -++ -++out: -++ sockopt_free_buf(&ctx); -++ return ret; -++} -++EXPORT_SYMBOL(__cgroup_bpf_run_filter_getsockopt); -++#endif -++ -++static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp, -++ size_t *lenp) -++{ -++ ssize_t tmp_ret = 0, ret; -++ -++ if (dir->header.parent) { -++ tmp_ret = sysctl_cpy_dir(dir->header.parent, bufp, lenp); -++ if (tmp_ret < 0) -++ return tmp_ret; -++ } -++ -++ ret = strscpy(*bufp, dir->header.ctl_table[0].procname, *lenp); -++ if (ret < 0) -++ return ret; -++ *bufp += ret; -++ *lenp -= ret; -++ ret += tmp_ret; -++ -++ /* Avoid leading slash. */ -++ if (!ret) -++ return ret; -++ -++ tmp_ret = strscpy(*bufp, "/", *lenp); -++ if (tmp_ret < 0) -++ return tmp_ret; -++ *bufp += tmp_ret; -++ *lenp -= tmp_ret; -++ -++ return ret + tmp_ret; -++} -++ -++BPF_CALL_4(bpf_sysctl_get_name, struct bpf_sysctl_kern *, ctx, char *, buf, -++ size_t, buf_len, u64, flags) -++{ -++ ssize_t tmp_ret = 0, ret; -++ -++ if (!buf) -++ return -EINVAL; -++ -++ if (!(flags & BPF_F_SYSCTL_BASE_NAME)) { -++ if (!ctx->head) -++ return -EINVAL; -++ tmp_ret = sysctl_cpy_dir(ctx->head->parent, &buf, &buf_len); -++ if (tmp_ret < 0) -++ return tmp_ret; -++ } -++ -++ ret = strscpy(buf, ctx->table->procname, buf_len); -++ -++ return ret < 0 ? ret : tmp_ret + ret; -++} -++ -++static const struct bpf_func_proto bpf_sysctl_get_name_proto = { -++ .func = bpf_sysctl_get_name, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_PTR_TO_MEM, -++ .arg3_type = ARG_CONST_SIZE, -++ .arg4_type = ARG_ANYTHING, -++}; -++ -++static int copy_sysctl_value(char *dst, size_t dst_len, char *src, -++ size_t src_len) -++{ -++ if (!dst) -++ return -EINVAL; -++ -++ if (!dst_len) -++ return -E2BIG; -++ -++ if (!src || !src_len) { -++ memset(dst, 0, dst_len); -++ return -EINVAL; -++ } -++ -++ memcpy(dst, src, min(dst_len, src_len)); -++ -++ if (dst_len > src_len) { -++ memset(dst + src_len, '\0', dst_len - src_len); -++ return src_len; -++ } -++ -++ dst[dst_len - 1] = '\0'; -++ -++ return -E2BIG; -++} -++ -++BPF_CALL_3(bpf_sysctl_get_current_value, struct bpf_sysctl_kern *, ctx, -++ char *, buf, size_t, buf_len) -++{ -++ return copy_sysctl_value(buf, buf_len, ctx->cur_val, ctx->cur_len); -++} -++ -++static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { -++ .func = bpf_sysctl_get_current_value, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg3_type = ARG_CONST_SIZE, -++}; -++ -++BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, -++ size_t, buf_len) -++{ -++ if (!ctx->write) { -++ if (buf && buf_len) -++ memset(buf, '\0', buf_len); -++ return -EINVAL; -++ } -++ return copy_sysctl_value(buf, buf_len, ctx->new_val, ctx->new_len); -++} -++ -++static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { -++ .func = bpf_sysctl_get_new_value, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg3_type = ARG_CONST_SIZE, -++}; -++ -++BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, -++ const char *, buf, size_t, buf_len) -++{ -++ if (!ctx->write || !ctx->new_val || !ctx->new_len || !buf || !buf_len) -++ return -EINVAL; -++ -++ if (buf_len > PAGE_SIZE - 1) -++ return -E2BIG; -++ -++ memcpy(ctx->new_val, buf, buf_len); -++ ctx->new_len = buf_len; -++ ctx->new_updated = 1; -++ -++ return 0; -++} -++ -++static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { -++ .func = bpf_sysctl_set_new_value, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_PTR_TO_MEM, -++ .arg3_type = ARG_CONST_SIZE, -++}; -++ -++static const struct bpf_func_proto * -++sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ case BPF_FUNC_strtol: -++ return &bpf_strtol_proto; -++ case BPF_FUNC_strtoul: -++ return &bpf_strtoul_proto; -++ case BPF_FUNC_sysctl_get_name: -++ return &bpf_sysctl_get_name_proto; -++ case BPF_FUNC_sysctl_get_current_value: -++ return &bpf_sysctl_get_current_value_proto; -++ case BPF_FUNC_sysctl_get_new_value: -++ return &bpf_sysctl_get_new_value_proto; -++ case BPF_FUNC_sysctl_set_new_value: -++ return &bpf_sysctl_set_new_value_proto; -++ default: -++ return cgroup_base_func_proto(func_id, prog); -++ } -++} -++ -++static bool sysctl_is_valid_access(int off, int size, enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ const int size_default = sizeof(__u32); -++ -++ if (off < 0 || off + size > sizeof(struct bpf_sysctl) || off % size) -++ return false; -++ -++ switch (off) { -++ case bpf_ctx_range(struct bpf_sysctl, write): -++ if (type != BPF_READ) -++ return false; -++ bpf_ctx_record_field_size(info, size_default); -++ return bpf_ctx_narrow_access_ok(off, size, size_default); -++ case bpf_ctx_range(struct bpf_sysctl, file_pos): -++ if (type == BPF_READ) { -++ bpf_ctx_record_field_size(info, size_default); -++ return bpf_ctx_narrow_access_ok(off, size, size_default); -++ } else { -++ return size == size_default; -++ } -++ default: -++ return false; -++ } -++} -++ -++static u32 sysctl_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ struct bpf_insn *insn = insn_buf; -++ u32 read_size; -++ -++ switch (si->off) { -++ case offsetof(struct bpf_sysctl, write): -++ *insn++ = BPF_LDX_MEM( -++ BPF_SIZE(si->code), si->dst_reg, si->src_reg, -++ bpf_target_off(struct bpf_sysctl_kern, write, -++ FIELD_SIZEOF(struct bpf_sysctl_kern, -++ write), -++ target_size)); -++ break; -++ case offsetof(struct bpf_sysctl, file_pos): -++ /* ppos is a pointer so it should be accessed via indirect -++ * loads and stores. Also for stores additional temporary -++ * register is used since neither src_reg nor dst_reg can be -++ * overridden. -++ */ -++ if (type == BPF_WRITE) { -++ int treg = BPF_REG_9; -++ -++ if (si->src_reg == treg || si->dst_reg == treg) -++ --treg; -++ if (si->src_reg == treg || si->dst_reg == treg) -++ --treg; -++ *insn++ = BPF_STX_MEM( -++ BPF_DW, si->dst_reg, treg, -++ offsetof(struct bpf_sysctl_kern, tmp_reg)); -++ *insn++ = BPF_LDX_MEM( -++ BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos), -++ treg, si->dst_reg, -++ offsetof(struct bpf_sysctl_kern, ppos)); -++ *insn++ = BPF_STX_MEM( -++ BPF_SIZEOF(u32), treg, si->src_reg, -++ bpf_ctx_narrow_access_offset( -++ 0, sizeof(u32), sizeof(loff_t))); -++ *insn++ = BPF_LDX_MEM( -++ BPF_DW, treg, si->dst_reg, -++ offsetof(struct bpf_sysctl_kern, tmp_reg)); -++ } else { -++ *insn++ = BPF_LDX_MEM( -++ BPF_FIELD_SIZEOF(struct bpf_sysctl_kern, ppos), -++ si->dst_reg, si->src_reg, -++ offsetof(struct bpf_sysctl_kern, ppos)); -++ read_size = bpf_size_to_bytes(BPF_SIZE(si->code)); -++ *insn++ = BPF_LDX_MEM( -++ BPF_SIZE(si->code), si->dst_reg, si->dst_reg, -++ bpf_ctx_narrow_access_offset( -++ 0, read_size, sizeof(loff_t))); -++ } -++ *target_size = sizeof(u32); -++ break; -++ } -++ -++ return insn - insn_buf; -++} -++ -++const struct bpf_verifier_ops cg_sysctl_verifier_ops = { -++ .get_func_proto = sysctl_func_proto, -++ .is_valid_access = sysctl_is_valid_access, -++ .convert_ctx_access = sysctl_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops cg_sysctl_prog_ops = { -++}; -++ -++static const struct bpf_func_proto * -++cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++#ifdef CONFIG_NET -++ case BPF_FUNC_sk_storage_get: -++ return &bpf_sk_storage_get_proto; -++ case BPF_FUNC_sk_storage_delete: -++ return &bpf_sk_storage_delete_proto; -++#endif -++#ifdef CONFIG_INET -++ case BPF_FUNC_tcp_sock: -++ return &bpf_tcp_sock_proto; -++#endif -++ default: -++ return cgroup_base_func_proto(func_id, prog); -++ } -++} -++ -++static bool cg_sockopt_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ const int size_default = sizeof(__u32); -++ -++ if (off < 0 || off >= sizeof(struct bpf_sockopt)) -++ return false; -++ -++ if (off % size != 0) -++ return false; -++ -++ if (type == BPF_WRITE) { -++ switch (off) { -++ case offsetof(struct bpf_sockopt, retval): -++ if (size != size_default) -++ return false; -++ return prog->expected_attach_type == -++ BPF_CGROUP_GETSOCKOPT; -++ case offsetof(struct bpf_sockopt, optname): -++ /* fallthrough */ -++ case offsetof(struct bpf_sockopt, level): -++ if (size != size_default) -++ return false; -++ return prog->expected_attach_type == -++ BPF_CGROUP_SETSOCKOPT; -++ case offsetof(struct bpf_sockopt, optlen): -++ return size == size_default; -++ default: -++ return false; -++ } -++ } -++ -++ switch (off) { -++ case offsetof(struct bpf_sockopt, sk): -++ if (size != sizeof(__u64)) -++ return false; -++ info->reg_type = PTR_TO_SOCKET; -++ break; -++ case offsetof(struct bpf_sockopt, optval): -++ if (size != sizeof(__u64)) -++ return false; -++ info->reg_type = PTR_TO_PACKET; -++ break; -++ case offsetof(struct bpf_sockopt, optval_end): -++ if (size != sizeof(__u64)) -++ return false; -++ info->reg_type = PTR_TO_PACKET_END; -++ break; -++ case offsetof(struct bpf_sockopt, retval): -++ if (size != size_default) -++ return false; -++ return prog->expected_attach_type == BPF_CGROUP_GETSOCKOPT; -++ default: -++ if (size != size_default) -++ return false; -++ break; -++ } -++ return true; -++} -++ -++#define CG_SOCKOPT_ACCESS_FIELD(T, F) \ -++ T(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, F), \ -++ si->dst_reg, si->src_reg, \ -++ offsetof(struct bpf_sockopt_kern, F)) -++ -++static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++{ -++ struct bpf_insn *insn = insn_buf; -++ -++ switch (si->off) { -++ case offsetof(struct bpf_sockopt, sk): -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, sk); -++ break; -++ case offsetof(struct bpf_sockopt, level): -++ if (type == BPF_WRITE) -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, level); -++ else -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, level); -++ break; -++ case offsetof(struct bpf_sockopt, optname): -++ if (type == BPF_WRITE) -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optname); -++ else -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optname); -++ break; -++ case offsetof(struct bpf_sockopt, optlen): -++ if (type == BPF_WRITE) -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, optlen); -++ else -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen); -++ break; -++ case offsetof(struct bpf_sockopt, retval): -++ if (type == BPF_WRITE) -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval); -++ else -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval); -++ break; -++ case offsetof(struct bpf_sockopt, optval): -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval); -++ break; -++ case offsetof(struct bpf_sockopt, optval_end): -++ *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval_end); -++ break; -++ } -++ -++ return insn - insn_buf; -++} -++ -++static int cg_sockopt_get_prologue(struct bpf_insn *insn_buf, -++ bool direct_write, -++ const struct bpf_prog *prog) -++{ -++ /* Nothing to do for sockopt argument. The data is kzalloc'ated. -++ */ -++ return 0; -++} -++ -++const struct bpf_verifier_ops cg_sockopt_verifier_ops = { -++ .get_func_proto = cg_sockopt_func_proto, -++ .is_valid_access = cg_sockopt_is_valid_access, -++ .convert_ctx_access = cg_sockopt_convert_ctx_access, -++ .gen_prologue = cg_sockopt_get_prologue, -++}; -++ -++const struct bpf_prog_ops cg_sockopt_prog_ops = { -++}; -+--- a/kernel/bpf/core.c -++++ b/kernel/bpf/core.c -+@@ -1,3 +1,4 @@ -++// SPDX-License-Identifier: GPL-2.0-or-later -+ /* -+ * Linux Socket Filter - Kernel level socket filtering -+ * -+@@ -12,21 +13,22 @@ -+ * Alexei Starovoitov -+ * Daniel Borkmann -+ * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of the GNU General Public License -+- * as published by the Free Software Foundation; either version -+- * 2 of the License, or (at your option) any later version. -+- * -+ * Andi Kleen - Fix a few bad bugs and races. -+ * Kris Katterjohn - Added many additional checks in bpf_check_classic() -+ */ -+ -++#include -+ #include -+ #include -+ #include -+ #include -+ #include -+ #include -++#include -++#include -++#include -++#include -++#include -+ -+ #include -+ -+@@ -47,6 +49,7 @@ -+ #define DST regs[insn->dst_reg] -+ #define SRC regs[insn->src_reg] -+ #define FP regs[BPF_REG_FP] -++#define AX regs[BPF_REG_AX] -+ #define ARG1 regs[BPF_REG_ARG1] -+ #define CTX regs[BPF_REG_CTX] -+ #define IMM insn->imm -+@@ -70,10 +73,9 @@ void *bpf_internal_load_pointer_neg_help -+ return NULL; -+ } -+ -+-struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) -++struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) -+ { -+- gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | -+- gfp_extra_flags; -++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; -+ struct bpf_prog_aux *aux; -+ struct bpf_prog *fp; -+ -+@@ -82,8 +84,6 @@ struct bpf_prog *bpf_prog_alloc(unsigned -+ if (fp == NULL) -+ return NULL; -+ -+- kmemcheck_annotate_bitfield(fp, meta); -+- -+ aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags); -+ if (aux == NULL) { -+ vfree(fp); -+@@ -93,30 +93,151 @@ struct bpf_prog *bpf_prog_alloc(unsigned -+ fp->pages = size / PAGE_SIZE; -+ fp->aux = aux; -+ fp->aux->prog = fp; -++ fp->jit_requested = ebpf_jit_enabled(); -++ -++ INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode); -+ -+ return fp; -+ } -++ -++struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) -++{ -++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; -++ struct bpf_prog *prog; -++ int cpu; -++ -++ prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); -++ if (!prog) -++ return NULL; -++ -++ prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); -++ if (!prog->aux->stats) { -++ kfree(prog->aux); -++ vfree(prog); -++ return NULL; -++ } -++ -++ for_each_possible_cpu(cpu) { -++ struct bpf_prog_stats *pstats; -++ -++ pstats = per_cpu_ptr(prog->aux->stats, cpu); -++ u64_stats_init(&pstats->syncp); -++ } -++ return prog; -++} -+ EXPORT_SYMBOL_GPL(bpf_prog_alloc); -+ -++int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) -++{ -++ if (!prog->aux->nr_linfo || !prog->jit_requested) -++ return 0; -++ -++ prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, -++ sizeof(*prog->aux->jited_linfo), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!prog->aux->jited_linfo) -++ return -ENOMEM; -++ -++ return 0; -++} -++ -++void bpf_prog_free_jited_linfo(struct bpf_prog *prog) -++{ -++ kfree(prog->aux->jited_linfo); -++ prog->aux->jited_linfo = NULL; -++} -++ -++void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog) -++{ -++ if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) -++ bpf_prog_free_jited_linfo(prog); -++} -++ -++/* The jit engine is responsible to provide an array -++ * for insn_off to the jited_off mapping (insn_to_jit_off). -++ * -++ * The idx to this array is the insn_off. Hence, the insn_off -++ * here is relative to the prog itself instead of the main prog. -++ * This array has one entry for each xlated bpf insn. -++ * -++ * jited_off is the byte off to the last byte of the jited insn. -++ * -++ * Hence, with -++ * insn_start: -++ * The first bpf insn off of the prog. The insn off -++ * here is relative to the main prog. -++ * e.g. if prog is a subprog, insn_start > 0 -++ * linfo_idx: -++ * The prog's idx to prog->aux->linfo and jited_linfo -++ * -++ * jited_linfo[linfo_idx] = prog->bpf_func -++ * -++ * For i > linfo_idx, -++ * -++ * jited_linfo[i] = prog->bpf_func + -++ * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] -++ */ -++void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, -++ const u32 *insn_to_jit_off) -++{ -++ u32 linfo_idx, insn_start, insn_end, nr_linfo, i; -++ const struct bpf_line_info *linfo; -++ void **jited_linfo; -++ -++ if (!prog->aux->jited_linfo) -++ /* Userspace did not provide linfo */ -++ return; -++ -++ linfo_idx = prog->aux->linfo_idx; -++ linfo = &prog->aux->linfo[linfo_idx]; -++ insn_start = linfo[0].insn_off; -++ insn_end = insn_start + prog->len; -++ -++ jited_linfo = &prog->aux->jited_linfo[linfo_idx]; -++ jited_linfo[0] = prog->bpf_func; -++ -++ nr_linfo = prog->aux->nr_linfo - linfo_idx; -++ -++ for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) -++ /* The verifier ensures that linfo[i].insn_off is -++ * strictly increasing -++ */ -++ jited_linfo[i] = prog->bpf_func + -++ insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; -++} -++ -++void bpf_prog_free_linfo(struct bpf_prog *prog) -++{ -++ bpf_prog_free_jited_linfo(prog); -++ kvfree(prog->aux->linfo); -++} -++ -+ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, -+ gfp_t gfp_extra_flags) -+ { -+- gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | -+- gfp_extra_flags; -++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; -+ struct bpf_prog *fp; -++ u32 pages, delta; -++ int ret; -+ -+ BUG_ON(fp_old == NULL); -+ -+ size = round_up(size, PAGE_SIZE); -+- if (size <= fp_old->pages * PAGE_SIZE) -++ pages = size / PAGE_SIZE; -++ if (pages <= fp_old->pages) -+ return fp_old; -+ -+- fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); -+- if (fp != NULL) { -+- kmemcheck_annotate_bitfield(fp, meta); -++ delta = pages - fp_old->pages; -++ ret = __bpf_prog_charge(fp_old->aux->user, delta); -++ if (ret) -++ return NULL; -+ -++ fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); -++ if (fp == NULL) { -++ __bpf_prog_uncharge(fp_old->aux->user, delta); -++ } else { -+ memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); -+- fp->pages = size / PAGE_SIZE; -++ fp->pages = pages; -+ fp->aux->prog = fp; -+ -+ /* We keep fp->aux from fp_old around in the new -+@@ -128,40 +249,578 @@ struct bpf_prog *bpf_prog_realloc(struct -+ -+ return fp; -+ } -+-EXPORT_SYMBOL_GPL(bpf_prog_realloc); -+ -+ void __bpf_prog_free(struct bpf_prog *fp) -+ { -+- kfree(fp->aux); -++ if (fp->aux) { -++ free_percpu(fp->aux->stats); -++ kfree(fp->aux); -++ } -+ vfree(fp); -+ } -+-EXPORT_SYMBOL_GPL(__bpf_prog_free); -++ -++int bpf_prog_calc_tag(struct bpf_prog *fp) -++{ -++ const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64); -++ u32 raw_size = bpf_prog_tag_scratch_size(fp); -++ u32 digest[SHA_DIGEST_WORDS]; -++ u32 ws[SHA_WORKSPACE_WORDS]; -++ u32 i, bsize, psize, blocks; -++ struct bpf_insn *dst; -++ bool was_ld_map; -++ u8 *raw, *todo; -++ __be32 *result; -++ __be64 *bits; -++ -++ raw = vmalloc(raw_size); -++ if (!raw) -++ return -ENOMEM; -++ -++ sha_init(digest); -++ memset(ws, 0, sizeof(ws)); -++ -++ /* We need to take out the map fd for the digest calculation -++ * since they are unstable from user space side. -++ */ -++ dst = (void *)raw; -++ for (i = 0, was_ld_map = false; i < fp->len; i++) { -++ dst[i] = fp->insnsi[i]; -++ if (!was_ld_map && -++ dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && -++ (dst[i].src_reg == BPF_PSEUDO_MAP_FD || -++ dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { -++ was_ld_map = true; -++ dst[i].imm = 0; -++ } else if (was_ld_map && -++ dst[i].code == 0 && -++ dst[i].dst_reg == 0 && -++ dst[i].src_reg == 0 && -++ dst[i].off == 0) { -++ was_ld_map = false; -++ dst[i].imm = 0; -++ } else { -++ was_ld_map = false; -++ } -++ } -++ -++ psize = bpf_prog_insn_size(fp); -++ memset(&raw[psize], 0, raw_size - psize); -++ raw[psize++] = 0x80; -++ -++ bsize = round_up(psize, SHA_MESSAGE_BYTES); -++ blocks = bsize / SHA_MESSAGE_BYTES; -++ todo = raw; -++ if (bsize - psize >= sizeof(__be64)) { -++ bits = (__be64 *)(todo + bsize - sizeof(__be64)); -++ } else { -++ bits = (__be64 *)(todo + bsize + bits_offset); -++ blocks++; -++ } -++ *bits = cpu_to_be64((psize - 1) << 3); -++ -++ while (blocks--) { -++ sha_transform(digest, todo, ws); -++ todo += SHA_MESSAGE_BYTES; -++ } -++ -++ result = (__force __be32 *)digest; -++ for (i = 0; i < SHA_DIGEST_WORDS; i++) -++ result[i] = cpu_to_be32(digest[i]); -++ memcpy(fp->tag, result, sizeof(fp->tag)); -++ -++ vfree(raw); -++ return 0; -++} -++ -++static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, -++ s32 end_new, s32 curr, const bool probe_pass) -++{ -++ const s64 imm_min = S32_MIN, imm_max = S32_MAX; -++ s32 delta = end_new - end_old; -++ s64 imm = insn->imm; -++ -++ if (curr < pos && curr + imm + 1 >= end_old) -++ imm += delta; -++ else if (curr >= end_new && curr + imm + 1 < end_new) -++ imm -= delta; -++ if (imm < imm_min || imm > imm_max) -++ return -ERANGE; -++ if (!probe_pass) -++ insn->imm = imm; -++ return 0; -++} -++ -++static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, -++ s32 end_new, s32 curr, const bool probe_pass) -++{ -++ const s32 off_min = S16_MIN, off_max = S16_MAX; -++ s32 delta = end_new - end_old; -++ s32 off = insn->off; -++ -++ if (curr < pos && curr + off + 1 >= end_old) -++ off += delta; -++ else if (curr >= end_new && curr + off + 1 < end_new) -++ off -= delta; -++ if (off < off_min || off > off_max) -++ return -ERANGE; -++ if (!probe_pass) -++ insn->off = off; -++ return 0; -++} -++ -++static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, -++ s32 end_new, const bool probe_pass) -++{ -++ u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); -++ struct bpf_insn *insn = prog->insnsi; -++ int ret = 0; -++ -++ for (i = 0; i < insn_cnt; i++, insn++) { -++ u8 code; -++ -++ /* In the probing pass we still operate on the original, -++ * unpatched image in order to check overflows before we -++ * do any other adjustments. Therefore skip the patchlet. -++ */ -++ if (probe_pass && i == pos) { -++ i = end_new; -++ insn = prog->insnsi + end_old; -++ } -++ code = insn->code; -++ if ((BPF_CLASS(code) != BPF_JMP && -++ BPF_CLASS(code) != BPF_JMP32) || -++ BPF_OP(code) == BPF_EXIT) -++ continue; -++ /* Adjust offset of jmps if we cross patch boundaries. */ -++ if (BPF_OP(code) == BPF_CALL) { -++ if (insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ ret = bpf_adj_delta_to_imm(insn, pos, end_old, -++ end_new, i, probe_pass); -++ } else { -++ ret = bpf_adj_delta_to_off(insn, pos, end_old, -++ end_new, i, probe_pass); -++ } -++ if (ret) -++ break; -++ } -++ -++ return ret; -++} -++ -++static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) -++{ -++ struct bpf_line_info *linfo; -++ u32 i, nr_linfo; -++ -++ nr_linfo = prog->aux->nr_linfo; -++ if (!nr_linfo || !delta) -++ return; -++ -++ linfo = prog->aux->linfo; -++ -++ for (i = 0; i < nr_linfo; i++) -++ if (off < linfo[i].insn_off) -++ break; -++ -++ /* Push all off < linfo[i].insn_off by delta */ -++ for (; i < nr_linfo; i++) -++ linfo[i].insn_off += delta; -++} -++ -++struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, -++ const struct bpf_insn *patch, u32 len) -++{ -++ u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; -++ const u32 cnt_max = S16_MAX; -++ struct bpf_prog *prog_adj; -++ int err; -++ -++ /* Since our patchlet doesn't expand the image, we're done. */ -++ if (insn_delta == 0) { -++ memcpy(prog->insnsi + off, patch, sizeof(*patch)); -++ return prog; -++ } -++ -++ insn_adj_cnt = prog->len + insn_delta; -++ -++ /* Reject anything that would potentially let the insn->off -++ * target overflow when we have excessive program expansions. -++ * We need to probe here before we do any reallocation where -++ * we afterwards may not fail anymore. -++ */ -++ if (insn_adj_cnt > cnt_max && -++ (err = bpf_adj_branches(prog, off, off + 1, off + len, true))) -++ return ERR_PTR(err); -++ -++ /* Several new instructions need to be inserted. Make room -++ * for them. Likely, there's no need for a new allocation as -++ * last page could have large enough tailroom. -++ */ -++ prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), -++ GFP_USER); -++ if (!prog_adj) -++ return ERR_PTR(-ENOMEM); -++ -++ prog_adj->len = insn_adj_cnt; -++ -++ /* Patching happens in 3 steps: -++ * -++ * 1) Move over tail of insnsi from next instruction onwards, -++ * so we can patch the single target insn with one or more -++ * new ones (patching is always from 1 to n insns, n > 0). -++ * 2) Inject new instructions at the target location. -++ * 3) Adjust branch offsets if necessary. -++ */ -++ insn_rest = insn_adj_cnt - off - len; -++ -++ memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, -++ sizeof(*patch) * insn_rest); -++ memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); -++ -++ /* We are guaranteed to not fail at this point, otherwise -++ * the ship has sailed to reverse to the original state. An -++ * overflow cannot happen at this point. -++ */ -++ BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); -++ -++ bpf_adj_linfo(prog_adj, off, insn_delta); -++ -++ return prog_adj; -++} -++ -++int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) -++{ -++ /* Branch offsets can't overflow when program is shrinking, no need -++ * to call bpf_adj_branches(..., true) here -++ */ -++ memmove(prog->insnsi + off, prog->insnsi + off + cnt, -++ sizeof(struct bpf_insn) * (prog->len - off - cnt)); -++ prog->len -= cnt; -++ -++ return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); -++} -++ -++static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) -++{ -++ int i; -++ -++ for (i = 0; i < fp->aux->func_cnt; i++) -++ bpf_prog_kallsyms_del(fp->aux->func[i]); -++} -++ -++void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) -++{ -++ bpf_prog_kallsyms_del_subprogs(fp); -++ bpf_prog_kallsyms_del(fp); -++} -+ -+ #ifdef CONFIG_BPF_JIT -++/* All BPF JIT sysctl knobs here. */ -++int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON); -++int bpf_jit_harden __read_mostly; -++int bpf_jit_kallsyms __read_mostly; -++long bpf_jit_limit __read_mostly; -++ -++static __always_inline void -++bpf_get_prog_addr_region(const struct bpf_prog *prog, -++ unsigned long *symbol_start, -++ unsigned long *symbol_end) -++{ -++ const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog); -++ unsigned long addr = (unsigned long)hdr; -++ -++ WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); -++ -++ *symbol_start = addr; -++ *symbol_end = addr + hdr->pages * PAGE_SIZE; -++} -++ -++void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) -++{ -++ const char *end = sym + KSYM_NAME_LEN; -++ const struct btf_type *type; -++ const char *func_name; -++ -++ BUILD_BUG_ON(sizeof("bpf_prog_") + -++ sizeof(prog->tag) * 2 + -++ /* name has been null terminated. -++ * We should need +1 for the '_' preceding -++ * the name. However, the null character -++ * is double counted between the name and the -++ * sizeof("bpf_prog_") above, so we omit -++ * the +1 here. -++ */ -++ sizeof(prog->aux->name) > KSYM_NAME_LEN); -++ -++ sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); -++ sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); -++ -++ /* prog->aux->name will be ignored if full btf name is available */ -++ if (prog->aux->func_info_cnt) { -++ type = btf_type_by_id(prog->aux->btf, -++ prog->aux->func_info[prog->aux->func_idx].type_id); -++ func_name = btf_name_by_offset(prog->aux->btf, type->name_off); -++ snprintf(sym, (size_t)(end - sym), "_%s", func_name); -++ return; -++ } -++ -++ if (prog->aux->name[0]) -++ snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); -++ else -++ *sym = 0; -++} -++ -++static __always_inline unsigned long -++bpf_get_prog_addr_start(struct latch_tree_node *n) -++{ -++ unsigned long symbol_start, symbol_end; -++ const struct bpf_prog_aux *aux; -++ -++ aux = container_of(n, struct bpf_prog_aux, ksym_tnode); -++ bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); -++ -++ return symbol_start; -++} -++ -++static __always_inline bool bpf_tree_less(struct latch_tree_node *a, -++ struct latch_tree_node *b) -++{ -++ return bpf_get_prog_addr_start(a) < bpf_get_prog_addr_start(b); -++} -++ -++static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) -++{ -++ unsigned long val = (unsigned long)key; -++ unsigned long symbol_start, symbol_end; -++ const struct bpf_prog_aux *aux; -++ -++ aux = container_of(n, struct bpf_prog_aux, ksym_tnode); -++ bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); -++ -++ if (val < symbol_start) -++ return -1; -++ if (val >= symbol_end) -++ return 1; -++ -++ return 0; -++} -++ -++static const struct latch_tree_ops bpf_tree_ops = { -++ .less = bpf_tree_less, -++ .comp = bpf_tree_comp, -++}; -++ -++static DEFINE_SPINLOCK(bpf_lock); -++static LIST_HEAD(bpf_kallsyms); -++static struct latch_tree_root bpf_tree __cacheline_aligned; -++ -++static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) -++{ -++ WARN_ON_ONCE(!list_empty(&aux->ksym_lnode)); -++ list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms); -++ latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); -++} -++ -++static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux) -++{ -++ if (list_empty(&aux->ksym_lnode)) -++ return; -++ -++ latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); -++ list_del_rcu(&aux->ksym_lnode); -++} -++ -++static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) -++{ -++ return fp->jited && !bpf_prog_was_classic(fp); -++} -++ -++static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) -++{ -++ return list_empty(&fp->aux->ksym_lnode) || -++ fp->aux->ksym_lnode.prev == LIST_POISON2; -++} -++ -++void bpf_prog_kallsyms_add(struct bpf_prog *fp) -++{ -++ if (!bpf_prog_kallsyms_candidate(fp) || -++ !capable(CAP_SYS_ADMIN)) -++ return; -++ -++ spin_lock_bh(&bpf_lock); -++ bpf_prog_ksym_node_add(fp->aux); -++ spin_unlock_bh(&bpf_lock); -++} -++ -++void bpf_prog_kallsyms_del(struct bpf_prog *fp) -++{ -++ if (!bpf_prog_kallsyms_candidate(fp)) -++ return; -++ -++ spin_lock_bh(&bpf_lock); -++ bpf_prog_ksym_node_del(fp->aux); -++ spin_unlock_bh(&bpf_lock); -++} -++ -++static struct bpf_prog *bpf_prog_kallsyms_find(unsigned long addr) -++{ -++ struct latch_tree_node *n; -++ -++ if (!bpf_jit_kallsyms_enabled()) -++ return NULL; -++ -++ n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); -++ return n ? -++ container_of(n, struct bpf_prog_aux, ksym_tnode)->prog : -++ NULL; -++} -++ -++const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, -++ unsigned long *off, char *sym) -++{ -++ unsigned long symbol_start, symbol_end; -++ struct bpf_prog *prog; -++ char *ret = NULL; -++ -++ rcu_read_lock(); -++ prog = bpf_prog_kallsyms_find(addr); -++ if (prog) { -++ bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end); -++ bpf_get_prog_name(prog, sym); -++ -++ ret = sym; -++ if (size) -++ *size = symbol_end - symbol_start; -++ if (off) -++ *off = addr - symbol_start; -++ } -++ rcu_read_unlock(); -++ -++ return ret; -++} -++ -++bool is_bpf_text_address(unsigned long addr) -++{ -++ bool ret; -++ -++ rcu_read_lock(); -++ ret = bpf_prog_kallsyms_find(addr) != NULL; -++ rcu_read_unlock(); -++ -++ return ret; -++} -++ -++int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, -++ char *sym) -++{ -++ struct bpf_prog_aux *aux; -++ unsigned int it = 0; -++ int ret = -ERANGE; -++ -++ if (!bpf_jit_kallsyms_enabled()) -++ return ret; -++ -++ rcu_read_lock(); -++ list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) { -++ if (it++ != symnum) -++ continue; -++ -++ bpf_get_prog_name(aux->prog, sym); -++ -++ *value = (unsigned long)aux->prog->bpf_func; -++ *type = BPF_SYM_ELF_TYPE; -++ -++ ret = 0; -++ break; -++ } -++ rcu_read_unlock(); -++ -++ return ret; -++} -++ -++static atomic_long_t bpf_jit_current; -++ -++/* Can be overridden by an arch's JIT compiler if it has a custom, -++ * dedicated BPF backend memory area, or if neither of the two -++ * below apply. -++ */ -++u64 __weak bpf_jit_alloc_exec_limit(void) -++{ -++#if defined(MODULES_VADDR) -++ return MODULES_END - MODULES_VADDR; -++#else -++ return VMALLOC_END - VMALLOC_START; -++#endif -++} -++ -++static int __init bpf_jit_charge_init(void) -++{ -++ /* Only used as heuristic here to derive limit. */ -++ bpf_jit_limit = min_t(u64, round_up(bpf_jit_alloc_exec_limit() >> 2, -++ PAGE_SIZE), LONG_MAX); -++ return 0; -++} -++pure_initcall(bpf_jit_charge_init); -++ -++static int bpf_jit_charge_modmem(u32 pages) -++{ -++ if (atomic_long_add_return(pages, &bpf_jit_current) > -++ (bpf_jit_limit >> PAGE_SHIFT)) { -++ if (!capable(CAP_SYS_ADMIN)) { -++ atomic_long_sub(pages, &bpf_jit_current); -++ return -EPERM; -++ } -++ } -++ -++ return 0; -++} -++ -++static void bpf_jit_uncharge_modmem(u32 pages) -++{ -++ atomic_long_sub(pages, &bpf_jit_current); -++} -++ -++void *__weak bpf_jit_alloc_exec(unsigned long size) -++{ -++ return module_alloc(size); -++} -++ -++void __weak bpf_jit_free_exec(void *addr) -++{ -++ module_memfree(addr); -++} -++ -+ struct bpf_binary_header * -+ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, -+ unsigned int alignment, -+ bpf_jit_fill_hole_t bpf_fill_ill_insns) -+ { -+ struct bpf_binary_header *hdr; -+- unsigned int size, hole, start; -++ u32 size, hole, start, pages; -+ -+ /* Most of BPF filters are really small, but if some of them -+ * fill a page, allow at least 128 extra bytes to insert a -+ * random section of illegal instructions. -+ */ -+ size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); -+- hdr = module_alloc(size); -+- if (hdr == NULL) -++ pages = size / PAGE_SIZE; -++ -++ if (bpf_jit_charge_modmem(pages)) -++ return NULL; -++ hdr = bpf_jit_alloc_exec(size); -++ if (!hdr) { -++ bpf_jit_uncharge_modmem(pages); -+ return NULL; -++ } -+ -+ /* Fill space with illegal/arch-dep instructions. */ -+ bpf_fill_ill_insns(hdr, size); -+ -+- hdr->pages = size / PAGE_SIZE; -++ hdr->pages = pages; -+ hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), -+ PAGE_SIZE - sizeof(*hdr)); -+- start = (prandom_u32() % hole) & ~(alignment - 1); -++ start = (get_random_int() % hole) & ~(alignment - 1); -+ -+ /* Leave a random number of instructions before BPF code. */ -+ *image_ptr = &hdr->image[start]; -+@@ -171,13 +830,301 @@ bpf_jit_binary_alloc(unsigned int progle -+ -+ void bpf_jit_binary_free(struct bpf_binary_header *hdr) -+ { -+- module_memfree(hdr); -++ u32 pages = hdr->pages; -++ -++ bpf_jit_free_exec(hdr); -++ bpf_jit_uncharge_modmem(pages); -++} -++ -++/* This symbol is only overridden by archs that have different -++ * requirements than the usual eBPF JITs, f.e. when they only -++ * implement cBPF JIT, do not set images read-only, etc. -++ */ -++void __weak bpf_jit_free(struct bpf_prog *fp) -++{ -++ if (fp->jited) { -++ struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); -++ -++ bpf_jit_binary_unlock_ro(hdr); -++ bpf_jit_binary_free(hdr); -++ -++ WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); -++ } -++ -++ bpf_prog_unlock_free(fp); -++} -++ -++int bpf_jit_get_func_addr(const struct bpf_prog *prog, -++ const struct bpf_insn *insn, bool extra_pass, -++ u64 *func_addr, bool *func_addr_fixed) -++{ -++ s16 off = insn->off; -++ s32 imm = insn->imm; -++ u8 *addr; -++ -++ *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; -++ if (!*func_addr_fixed) { -++ /* Place-holder address till the last pass has collected -++ * all addresses for JITed subprograms in which case we -++ * can pick them up from prog->aux. -++ */ -++ if (!extra_pass) -++ addr = NULL; -++ else if (prog->aux->func && -++ off >= 0 && off < prog->aux->func_cnt) -++ addr = (u8 *)prog->aux->func[off]->bpf_func; -++ else -++ return -EINVAL; -++ } else { -++ /* Address of a BPF helper call. Since part of the core -++ * kernel, it's always at a fixed location. __bpf_call_base -++ * and the helper with imm relative to it are both in core -++ * kernel. -++ */ -++ addr = (u8 *)__bpf_call_base + imm; -++ } -++ -++ *func_addr = (unsigned long)addr; -++ return 0; -++} -++ -++static int bpf_jit_blind_insn(const struct bpf_insn *from, -++ const struct bpf_insn *aux, -++ struct bpf_insn *to_buff, -++ bool emit_zext) -++{ -++ struct bpf_insn *to = to_buff; -++ u32 imm_rnd = get_random_int(); -++ s16 off; -++ -++ BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); -++ BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); -++ -++ /* Constraints on AX register: -++ * -++ * AX register is inaccessible from user space. It is mapped in -++ * all JITs, and used here for constant blinding rewrites. It is -++ * typically "stateless" meaning its contents are only valid within -++ * the executed instruction, but not across several instructions. -++ * There are a few exceptions however which are further detailed -++ * below. -++ * -++ * Constant blinding is only used by JITs, not in the interpreter. -++ * The interpreter uses AX in some occasions as a local temporary -++ * register e.g. in DIV or MOD instructions. -++ * -++ * In restricted circumstances, the verifier can also use the AX -++ * register for rewrites as long as they do not interfere with -++ * the above cases! -++ */ -++ if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) -++ goto out; -++ -++ if (from->imm == 0 && -++ (from->code == (BPF_ALU | BPF_MOV | BPF_K) || -++ from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { -++ *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); -++ goto out; -++ } -++ -++ switch (from->code) { -++ case BPF_ALU | BPF_ADD | BPF_K: -++ case BPF_ALU | BPF_SUB | BPF_K: -++ case BPF_ALU | BPF_AND | BPF_K: -++ case BPF_ALU | BPF_OR | BPF_K: -++ case BPF_ALU | BPF_XOR | BPF_K: -++ case BPF_ALU | BPF_MUL | BPF_K: -++ case BPF_ALU | BPF_MOV | BPF_K: -++ case BPF_ALU | BPF_DIV | BPF_K: -++ case BPF_ALU | BPF_MOD | BPF_K: -++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); -++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX); -++ break; -++ -++ case BPF_ALU64 | BPF_ADD | BPF_K: -++ case BPF_ALU64 | BPF_SUB | BPF_K: -++ case BPF_ALU64 | BPF_AND | BPF_K: -++ case BPF_ALU64 | BPF_OR | BPF_K: -++ case BPF_ALU64 | BPF_XOR | BPF_K: -++ case BPF_ALU64 | BPF_MUL | BPF_K: -++ case BPF_ALU64 | BPF_MOV | BPF_K: -++ case BPF_ALU64 | BPF_DIV | BPF_K: -++ case BPF_ALU64 | BPF_MOD | BPF_K: -++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); -++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX); -++ break; -++ -++ case BPF_JMP | BPF_JEQ | BPF_K: -++ case BPF_JMP | BPF_JNE | BPF_K: -++ case BPF_JMP | BPF_JGT | BPF_K: -++ case BPF_JMP | BPF_JLT | BPF_K: -++ case BPF_JMP | BPF_JGE | BPF_K: -++ case BPF_JMP | BPF_JLE | BPF_K: -++ case BPF_JMP | BPF_JSGT | BPF_K: -++ case BPF_JMP | BPF_JSLT | BPF_K: -++ case BPF_JMP | BPF_JSGE | BPF_K: -++ case BPF_JMP | BPF_JSLE | BPF_K: -++ case BPF_JMP | BPF_JSET | BPF_K: -++ /* Accommodate for extra offset in case of a backjump. */ -++ off = from->off; -++ if (off < 0) -++ off -= 2; -++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); -++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); -++ break; -++ -++ case BPF_JMP32 | BPF_JEQ | BPF_K: -++ case BPF_JMP32 | BPF_JNE | BPF_K: -++ case BPF_JMP32 | BPF_JGT | BPF_K: -++ case BPF_JMP32 | BPF_JLT | BPF_K: -++ case BPF_JMP32 | BPF_JGE | BPF_K: -++ case BPF_JMP32 | BPF_JLE | BPF_K: -++ case BPF_JMP32 | BPF_JSGT | BPF_K: -++ case BPF_JMP32 | BPF_JSLT | BPF_K: -++ case BPF_JMP32 | BPF_JSGE | BPF_K: -++ case BPF_JMP32 | BPF_JSLE | BPF_K: -++ case BPF_JMP32 | BPF_JSET | BPF_K: -++ /* Accommodate for extra offset in case of a backjump. */ -++ off = from->off; -++ if (off < 0) -++ off -= 2; -++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); -++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, -++ off); -++ break; -++ -++ case BPF_LD | BPF_IMM | BPF_DW: -++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); -++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); -++ *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); -++ break; -++ case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ -++ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); -++ *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ if (emit_zext) -++ *to++ = BPF_ZEXT_REG(BPF_REG_AX); -++ *to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); -++ break; -++ -++ case BPF_ST | BPF_MEM | BPF_DW: -++ case BPF_ST | BPF_MEM | BPF_W: -++ case BPF_ST | BPF_MEM | BPF_H: -++ case BPF_ST | BPF_MEM | BPF_B: -++ *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); -++ *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); -++ *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); -++ break; -++ } -++out: -++ return to - to_buff; -++} -++ -++static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, -++ gfp_t gfp_extra_flags) -++{ -++ gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; -++ struct bpf_prog *fp; -++ -++ fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL); -++ if (fp != NULL) { -++ /* aux->prog still points to the fp_other one, so -++ * when promoting the clone to the real program, -++ * this still needs to be adapted. -++ */ -++ memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); -++ } -++ -++ return fp; -++} -++ -++static void bpf_prog_clone_free(struct bpf_prog *fp) -++{ -++ /* aux was stolen by the other clone, so we cannot free -++ * it from this path! It will be freed eventually by the -++ * other program on release. -++ * -++ * At this point, we don't need a deferred release since -++ * clone is guaranteed to not be locked. -++ */ -++ fp->aux = NULL; -++ __bpf_prog_free(fp); -++} -++ -++void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) -++{ -++ /* We have to repoint aux->prog to self, as we don't -++ * know whether fp here is the clone or the original. -++ */ -++ fp->aux->prog = fp; -++ bpf_prog_clone_free(fp_other); -++} -++ -++struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) -++{ -++ struct bpf_insn insn_buff[16], aux[2]; -++ struct bpf_prog *clone, *tmp; -++ int insn_delta, insn_cnt; -++ struct bpf_insn *insn; -++ int i, rewritten; -++ -++ if (!bpf_jit_blinding_enabled(prog) || prog->blinded) -++ return prog; -++ -++ clone = bpf_prog_clone_create(prog, GFP_USER); -++ if (!clone) -++ return ERR_PTR(-ENOMEM); -++ -++ insn_cnt = clone->len; -++ insn = clone->insnsi; -++ -++ for (i = 0; i < insn_cnt; i++, insn++) { -++ /* We temporarily need to hold the original ld64 insn -++ * so that we can still access the first part in the -++ * second blinding run. -++ */ -++ if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && -++ insn[1].code == 0) -++ memcpy(aux, insn, sizeof(aux)); -++ -++ rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, -++ clone->aux->verifier_zext); -++ if (!rewritten) -++ continue; -++ -++ tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); -++ if (IS_ERR(tmp)) { -++ /* Patching may have repointed aux->prog during -++ * realloc from the original one, so we need to -++ * fix it up here on error. -++ */ -++ bpf_jit_prog_release_other(prog, clone); -++ return tmp; -++ } -++ -++ clone = tmp; -++ insn_delta = rewritten - 1; -++ -++ /* Walk new program and skip insns we just inserted. */ -++ insn = clone->insnsi + i + insn_delta; -++ insn_cnt += insn_delta; -++ i += insn_delta; -++ } -++ -++ clone->blinded = 1; -++ return clone; -+ } -+ #endif /* CONFIG_BPF_JIT */ -+ -+ /* Base function for offset calculation. Needs to go into .text section, -+ * therefore keeping it non-static as well; will also be used by JITs -+- * anyway later on, so do not let the compiler omit it. -++ * anyway later on, so do not let the compiler omit it. This also needs -++ * to go into kallsyms for correlation from e.g. bpftool, so naming -++ * must not change. -+ */ -+ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -+ { -+@@ -185,157 +1132,243 @@ noinline u64 __bpf_call_base(u64 r1, u64 -+ } -+ EXPORT_SYMBOL_GPL(__bpf_call_base); -+ -++/* All UAPI available opcodes. */ -++#define BPF_INSN_MAP(INSN_2, INSN_3) \ -++ /* 32 bit ALU operations. */ \ -++ /* Register based. */ \ -++ INSN_3(ALU, ADD, X), \ -++ INSN_3(ALU, SUB, X), \ -++ INSN_3(ALU, AND, X), \ -++ INSN_3(ALU, OR, X), \ -++ INSN_3(ALU, LSH, X), \ -++ INSN_3(ALU, RSH, X), \ -++ INSN_3(ALU, XOR, X), \ -++ INSN_3(ALU, MUL, X), \ -++ INSN_3(ALU, MOV, X), \ -++ INSN_3(ALU, ARSH, X), \ -++ INSN_3(ALU, DIV, X), \ -++ INSN_3(ALU, MOD, X), \ -++ INSN_2(ALU, NEG), \ -++ INSN_3(ALU, END, TO_BE), \ -++ INSN_3(ALU, END, TO_LE), \ -++ /* Immediate based. */ \ -++ INSN_3(ALU, ADD, K), \ -++ INSN_3(ALU, SUB, K), \ -++ INSN_3(ALU, AND, K), \ -++ INSN_3(ALU, OR, K), \ -++ INSN_3(ALU, LSH, K), \ -++ INSN_3(ALU, RSH, K), \ -++ INSN_3(ALU, XOR, K), \ -++ INSN_3(ALU, MUL, K), \ -++ INSN_3(ALU, MOV, K), \ -++ INSN_3(ALU, ARSH, K), \ -++ INSN_3(ALU, DIV, K), \ -++ INSN_3(ALU, MOD, K), \ -++ /* 64 bit ALU operations. */ \ -++ /* Register based. */ \ -++ INSN_3(ALU64, ADD, X), \ -++ INSN_3(ALU64, SUB, X), \ -++ INSN_3(ALU64, AND, X), \ -++ INSN_3(ALU64, OR, X), \ -++ INSN_3(ALU64, LSH, X), \ -++ INSN_3(ALU64, RSH, X), \ -++ INSN_3(ALU64, XOR, X), \ -++ INSN_3(ALU64, MUL, X), \ -++ INSN_3(ALU64, MOV, X), \ -++ INSN_3(ALU64, ARSH, X), \ -++ INSN_3(ALU64, DIV, X), \ -++ INSN_3(ALU64, MOD, X), \ -++ INSN_2(ALU64, NEG), \ -++ /* Immediate based. */ \ -++ INSN_3(ALU64, ADD, K), \ -++ INSN_3(ALU64, SUB, K), \ -++ INSN_3(ALU64, AND, K), \ -++ INSN_3(ALU64, OR, K), \ -++ INSN_3(ALU64, LSH, K), \ -++ INSN_3(ALU64, RSH, K), \ -++ INSN_3(ALU64, XOR, K), \ -++ INSN_3(ALU64, MUL, K), \ -++ INSN_3(ALU64, MOV, K), \ -++ INSN_3(ALU64, ARSH, K), \ -++ INSN_3(ALU64, DIV, K), \ -++ INSN_3(ALU64, MOD, K), \ -++ /* Call instruction. */ \ -++ INSN_2(JMP, CALL), \ -++ /* Exit instruction. */ \ -++ INSN_2(JMP, EXIT), \ -++ /* 32-bit Jump instructions. */ \ -++ /* Register based. */ \ -++ INSN_3(JMP32, JEQ, X), \ -++ INSN_3(JMP32, JNE, X), \ -++ INSN_3(JMP32, JGT, X), \ -++ INSN_3(JMP32, JLT, X), \ -++ INSN_3(JMP32, JGE, X), \ -++ INSN_3(JMP32, JLE, X), \ -++ INSN_3(JMP32, JSGT, X), \ -++ INSN_3(JMP32, JSLT, X), \ -++ INSN_3(JMP32, JSGE, X), \ -++ INSN_3(JMP32, JSLE, X), \ -++ INSN_3(JMP32, JSET, X), \ -++ /* Immediate based. */ \ -++ INSN_3(JMP32, JEQ, K), \ -++ INSN_3(JMP32, JNE, K), \ -++ INSN_3(JMP32, JGT, K), \ -++ INSN_3(JMP32, JLT, K), \ -++ INSN_3(JMP32, JGE, K), \ -++ INSN_3(JMP32, JLE, K), \ -++ INSN_3(JMP32, JSGT, K), \ -++ INSN_3(JMP32, JSLT, K), \ -++ INSN_3(JMP32, JSGE, K), \ -++ INSN_3(JMP32, JSLE, K), \ -++ INSN_3(JMP32, JSET, K), \ -++ /* Jump instructions. */ \ -++ /* Register based. */ \ -++ INSN_3(JMP, JEQ, X), \ -++ INSN_3(JMP, JNE, X), \ -++ INSN_3(JMP, JGT, X), \ -++ INSN_3(JMP, JLT, X), \ -++ INSN_3(JMP, JGE, X), \ -++ INSN_3(JMP, JLE, X), \ -++ INSN_3(JMP, JSGT, X), \ -++ INSN_3(JMP, JSLT, X), \ -++ INSN_3(JMP, JSGE, X), \ -++ INSN_3(JMP, JSLE, X), \ -++ INSN_3(JMP, JSET, X), \ -++ /* Immediate based. */ \ -++ INSN_3(JMP, JEQ, K), \ -++ INSN_3(JMP, JNE, K), \ -++ INSN_3(JMP, JGT, K), \ -++ INSN_3(JMP, JLT, K), \ -++ INSN_3(JMP, JGE, K), \ -++ INSN_3(JMP, JLE, K), \ -++ INSN_3(JMP, JSGT, K), \ -++ INSN_3(JMP, JSLT, K), \ -++ INSN_3(JMP, JSGE, K), \ -++ INSN_3(JMP, JSLE, K), \ -++ INSN_3(JMP, JSET, K), \ -++ INSN_2(JMP, JA), \ -++ /* Store instructions. */ \ -++ /* Register based. */ \ -++ INSN_3(STX, MEM, B), \ -++ INSN_3(STX, MEM, H), \ -++ INSN_3(STX, MEM, W), \ -++ INSN_3(STX, MEM, DW), \ -++ INSN_3(STX, XADD, W), \ -++ INSN_3(STX, XADD, DW), \ -++ /* Immediate based. */ \ -++ INSN_3(ST, MEM, B), \ -++ INSN_3(ST, MEM, H), \ -++ INSN_3(ST, MEM, W), \ -++ INSN_3(ST, MEM, DW), \ -++ /* Load instructions. */ \ -++ /* Register based. */ \ -++ INSN_3(LDX, MEM, B), \ -++ INSN_3(LDX, MEM, H), \ -++ INSN_3(LDX, MEM, W), \ -++ INSN_3(LDX, MEM, DW), \ -++ /* Immediate based. */ \ -++ INSN_3(LD, IMM, DW) -++ -++bool bpf_opcode_in_insntable(u8 code) -++{ -++#define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true -++#define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true -++ static const bool public_insntable[256] = { -++ [0 ... 255] = false, -++ /* Now overwrite non-defaults ... */ -++ BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), -++ /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ -++ [BPF_LD | BPF_ABS | BPF_B] = true, -++ [BPF_LD | BPF_ABS | BPF_H] = true, -++ [BPF_LD | BPF_ABS | BPF_W] = true, -++ [BPF_LD | BPF_IND | BPF_B] = true, -++ [BPF_LD | BPF_IND | BPF_H] = true, -++ [BPF_LD | BPF_IND | BPF_W] = true, -++ }; -++#undef BPF_INSN_3_TBL -++#undef BPF_INSN_2_TBL -++ return public_insntable[code]; -++} -++ -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -+ /** -+ * __bpf_prog_run - run eBPF program on a given context -+- * @ctx: is the data we are operating on -++ * @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers -+ * @insn: is the array of eBPF instructions -++ * @stack: is the eBPF storage stack -+ * -+ * Decode and execute eBPF instructions. -+ */ -+-static unsigned int __bpf_prog_run(void *ctx, const struct bpf_insn *insn) -++static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) -+ { -+- u64 stack[MAX_BPF_STACK / sizeof(u64)]; -+- u64 regs[MAX_BPF_REG], tmp; -+- static const void *jumptable[256] = { -++#define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y -++#define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z -++ static const void * const jumptable[256] = { -+ [0 ... 255] = &&default_label, -+ /* Now overwrite non-defaults ... */ -+- /* 32 bit ALU operations */ -+- [BPF_ALU | BPF_ADD | BPF_X] = &&ALU_ADD_X, -+- [BPF_ALU | BPF_ADD | BPF_K] = &&ALU_ADD_K, -+- [BPF_ALU | BPF_SUB | BPF_X] = &&ALU_SUB_X, -+- [BPF_ALU | BPF_SUB | BPF_K] = &&ALU_SUB_K, -+- [BPF_ALU | BPF_AND | BPF_X] = &&ALU_AND_X, -+- [BPF_ALU | BPF_AND | BPF_K] = &&ALU_AND_K, -+- [BPF_ALU | BPF_OR | BPF_X] = &&ALU_OR_X, -+- [BPF_ALU | BPF_OR | BPF_K] = &&ALU_OR_K, -+- [BPF_ALU | BPF_LSH | BPF_X] = &&ALU_LSH_X, -+- [BPF_ALU | BPF_LSH | BPF_K] = &&ALU_LSH_K, -+- [BPF_ALU | BPF_RSH | BPF_X] = &&ALU_RSH_X, -+- [BPF_ALU | BPF_RSH | BPF_K] = &&ALU_RSH_K, -+- [BPF_ALU | BPF_XOR | BPF_X] = &&ALU_XOR_X, -+- [BPF_ALU | BPF_XOR | BPF_K] = &&ALU_XOR_K, -+- [BPF_ALU | BPF_MUL | BPF_X] = &&ALU_MUL_X, -+- [BPF_ALU | BPF_MUL | BPF_K] = &&ALU_MUL_K, -+- [BPF_ALU | BPF_MOV | BPF_X] = &&ALU_MOV_X, -+- [BPF_ALU | BPF_MOV | BPF_K] = &&ALU_MOV_K, -+- [BPF_ALU | BPF_DIV | BPF_X] = &&ALU_DIV_X, -+- [BPF_ALU | BPF_DIV | BPF_K] = &&ALU_DIV_K, -+- [BPF_ALU | BPF_MOD | BPF_X] = &&ALU_MOD_X, -+- [BPF_ALU | BPF_MOD | BPF_K] = &&ALU_MOD_K, -+- [BPF_ALU | BPF_NEG] = &&ALU_NEG, -+- [BPF_ALU | BPF_END | BPF_TO_BE] = &&ALU_END_TO_BE, -+- [BPF_ALU | BPF_END | BPF_TO_LE] = &&ALU_END_TO_LE, -+- /* 64 bit ALU operations */ -+- [BPF_ALU64 | BPF_ADD | BPF_X] = &&ALU64_ADD_X, -+- [BPF_ALU64 | BPF_ADD | BPF_K] = &&ALU64_ADD_K, -+- [BPF_ALU64 | BPF_SUB | BPF_X] = &&ALU64_SUB_X, -+- [BPF_ALU64 | BPF_SUB | BPF_K] = &&ALU64_SUB_K, -+- [BPF_ALU64 | BPF_AND | BPF_X] = &&ALU64_AND_X, -+- [BPF_ALU64 | BPF_AND | BPF_K] = &&ALU64_AND_K, -+- [BPF_ALU64 | BPF_OR | BPF_X] = &&ALU64_OR_X, -+- [BPF_ALU64 | BPF_OR | BPF_K] = &&ALU64_OR_K, -+- [BPF_ALU64 | BPF_LSH | BPF_X] = &&ALU64_LSH_X, -+- [BPF_ALU64 | BPF_LSH | BPF_K] = &&ALU64_LSH_K, -+- [BPF_ALU64 | BPF_RSH | BPF_X] = &&ALU64_RSH_X, -+- [BPF_ALU64 | BPF_RSH | BPF_K] = &&ALU64_RSH_K, -+- [BPF_ALU64 | BPF_XOR | BPF_X] = &&ALU64_XOR_X, -+- [BPF_ALU64 | BPF_XOR | BPF_K] = &&ALU64_XOR_K, -+- [BPF_ALU64 | BPF_MUL | BPF_X] = &&ALU64_MUL_X, -+- [BPF_ALU64 | BPF_MUL | BPF_K] = &&ALU64_MUL_K, -+- [BPF_ALU64 | BPF_MOV | BPF_X] = &&ALU64_MOV_X, -+- [BPF_ALU64 | BPF_MOV | BPF_K] = &&ALU64_MOV_K, -+- [BPF_ALU64 | BPF_ARSH | BPF_X] = &&ALU64_ARSH_X, -+- [BPF_ALU64 | BPF_ARSH | BPF_K] = &&ALU64_ARSH_K, -+- [BPF_ALU64 | BPF_DIV | BPF_X] = &&ALU64_DIV_X, -+- [BPF_ALU64 | BPF_DIV | BPF_K] = &&ALU64_DIV_K, -+- [BPF_ALU64 | BPF_MOD | BPF_X] = &&ALU64_MOD_X, -+- [BPF_ALU64 | BPF_MOD | BPF_K] = &&ALU64_MOD_K, -+- [BPF_ALU64 | BPF_NEG] = &&ALU64_NEG, -+- /* Call instruction */ -+- [BPF_JMP | BPF_CALL] = &&JMP_CALL, -+- [BPF_JMP | BPF_CALL | BPF_X] = &&JMP_TAIL_CALL, -+- /* Jumps */ -+- [BPF_JMP | BPF_JA] = &&JMP_JA, -+- [BPF_JMP | BPF_JEQ | BPF_X] = &&JMP_JEQ_X, -+- [BPF_JMP | BPF_JEQ | BPF_K] = &&JMP_JEQ_K, -+- [BPF_JMP | BPF_JNE | BPF_X] = &&JMP_JNE_X, -+- [BPF_JMP | BPF_JNE | BPF_K] = &&JMP_JNE_K, -+- [BPF_JMP | BPF_JGT | BPF_X] = &&JMP_JGT_X, -+- [BPF_JMP | BPF_JGT | BPF_K] = &&JMP_JGT_K, -+- [BPF_JMP | BPF_JGE | BPF_X] = &&JMP_JGE_X, -+- [BPF_JMP | BPF_JGE | BPF_K] = &&JMP_JGE_K, -+- [BPF_JMP | BPF_JSGT | BPF_X] = &&JMP_JSGT_X, -+- [BPF_JMP | BPF_JSGT | BPF_K] = &&JMP_JSGT_K, -+- [BPF_JMP | BPF_JSGE | BPF_X] = &&JMP_JSGE_X, -+- [BPF_JMP | BPF_JSGE | BPF_K] = &&JMP_JSGE_K, -+- [BPF_JMP | BPF_JSET | BPF_X] = &&JMP_JSET_X, -+- [BPF_JMP | BPF_JSET | BPF_K] = &&JMP_JSET_K, -+- /* Program return */ -+- [BPF_JMP | BPF_EXIT] = &&JMP_EXIT, -+- /* Store instructions */ -+- [BPF_STX | BPF_MEM | BPF_B] = &&STX_MEM_B, -+- [BPF_STX | BPF_MEM | BPF_H] = &&STX_MEM_H, -+- [BPF_STX | BPF_MEM | BPF_W] = &&STX_MEM_W, -+- [BPF_STX | BPF_MEM | BPF_DW] = &&STX_MEM_DW, -+- [BPF_STX | BPF_XADD | BPF_W] = &&STX_XADD_W, -+- [BPF_STX | BPF_XADD | BPF_DW] = &&STX_XADD_DW, -+- [BPF_ST | BPF_MEM | BPF_B] = &&ST_MEM_B, -+- [BPF_ST | BPF_MEM | BPF_H] = &&ST_MEM_H, -+- [BPF_ST | BPF_MEM | BPF_W] = &&ST_MEM_W, -+- [BPF_ST | BPF_MEM | BPF_DW] = &&ST_MEM_DW, -+- /* Load instructions */ -+- [BPF_LDX | BPF_MEM | BPF_B] = &&LDX_MEM_B, -+- [BPF_LDX | BPF_MEM | BPF_H] = &&LDX_MEM_H, -+- [BPF_LDX | BPF_MEM | BPF_W] = &&LDX_MEM_W, -+- [BPF_LDX | BPF_MEM | BPF_DW] = &&LDX_MEM_DW, -+- [BPF_LD | BPF_ABS | BPF_W] = &&LD_ABS_W, -+- [BPF_LD | BPF_ABS | BPF_H] = &&LD_ABS_H, -+- [BPF_LD | BPF_ABS | BPF_B] = &&LD_ABS_B, -+- [BPF_LD | BPF_IND | BPF_W] = &&LD_IND_W, -+- [BPF_LD | BPF_IND | BPF_H] = &&LD_IND_H, -+- [BPF_LD | BPF_IND | BPF_B] = &&LD_IND_B, -+- [BPF_LD | BPF_IMM | BPF_DW] = &&LD_IMM_DW, -++ BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), -++ /* Non-UAPI available opcodes. */ -++ [BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, -++ [BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, -+ }; -++#undef BPF_INSN_3_LBL -++#undef BPF_INSN_2_LBL -+ u32 tail_call_cnt = 0; -+- void *ptr; -+- int off; -+ -+ #define CONT ({ insn++; goto select_insn; }) -+ #define CONT_JMP ({ insn++; goto select_insn; }) -+ -+- FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; -+- ARG1 = (u64) (unsigned long) ctx; -+- -+- /* Registers used in classic BPF programs need to be reset first. */ -+- regs[BPF_REG_A] = 0; -+- regs[BPF_REG_X] = 0; -+- -+ select_insn: -+ goto *jumptable[insn->code]; -+ -+- /* ALU */ -+-#define ALU(OPCODE, OP) \ -+- ALU64_##OPCODE##_X: \ -+- DST = DST OP SRC; \ -+- CONT; \ -+- ALU_##OPCODE##_X: \ -+- DST = (u32) DST OP (u32) SRC; \ -+- CONT; \ -+- ALU64_##OPCODE##_K: \ -+- DST = DST OP IMM; \ -+- CONT; \ -+- ALU_##OPCODE##_K: \ -+- DST = (u32) DST OP (u32) IMM; \ -++ /* Explicitly mask the register-based shift amounts with 63 or 31 -++ * to avoid undefined behavior. Normally this won't affect the -++ * generated code, for example, in case of native 64 bit archs such -++ * as x86-64 or arm64, the compiler is optimizing the AND away for -++ * the interpreter. In case of JITs, each of the JIT backends compiles -++ * the BPF shift operations to machine instructions which produce -++ * implementation-defined results in such a case; the resulting -++ * contents of the register may be arbitrary, but program behaviour -++ * as a whole remains defined. In other words, in case of JIT backends, -++ * the AND must /not/ be added to the emitted LSH/RSH/ARSH translation. -++ */ -++ /* ALU (shifts) */ -++#define SHT(OPCODE, OP) \ -++ ALU64_##OPCODE##_X: \ -++ DST = DST OP (SRC & 63); \ -++ CONT; \ -++ ALU_##OPCODE##_X: \ -++ DST = (u32) DST OP ((u32) SRC & 31); \ -++ CONT; \ -++ ALU64_##OPCODE##_K: \ -++ DST = DST OP IMM; \ -++ CONT; \ -++ ALU_##OPCODE##_K: \ -++ DST = (u32) DST OP (u32) IMM; \ -++ CONT; -++ /* ALU (rest) */ -++#define ALU(OPCODE, OP) \ -++ ALU64_##OPCODE##_X: \ -++ DST = DST OP SRC; \ -++ CONT; \ -++ ALU_##OPCODE##_X: \ -++ DST = (u32) DST OP (u32) SRC; \ -++ CONT; \ -++ ALU64_##OPCODE##_K: \ -++ DST = DST OP IMM; \ -++ CONT; \ -++ ALU_##OPCODE##_K: \ -++ DST = (u32) DST OP (u32) IMM; \ -+ CONT; -+- -+ ALU(ADD, +) -+ ALU(SUB, -) -+ ALU(AND, &) -+ ALU(OR, |) -+- ALU(LSH, <<) -+- ALU(RSH, >>) -+ ALU(XOR, ^) -+ ALU(MUL, *) -++ SHT(LSH, <<) -++ SHT(RSH, >>) -++#undef SHT -+ #undef ALU -+ ALU_NEG: -+ DST = (u32) -DST; -+@@ -359,51 +1392,49 @@ select_insn: -+ DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; -+ insn++; -+ CONT; -++ ALU_ARSH_X: -++ DST = (u64) (u32) (((s32) DST) >> (SRC & 31)); -++ CONT; -++ ALU_ARSH_K: -++ DST = (u64) (u32) (((s32) DST) >> IMM); -++ CONT; -+ ALU64_ARSH_X: -+- (*(s64 *) &DST) >>= SRC; -++ (*(s64 *) &DST) >>= (SRC & 63); -+ CONT; -+ ALU64_ARSH_K: -+ (*(s64 *) &DST) >>= IMM; -+ CONT; -+ ALU64_MOD_X: -+- if (unlikely(SRC == 0)) -+- return 0; -+- div64_u64_rem(DST, SRC, &tmp); -+- DST = tmp; -++ div64_u64_rem(DST, SRC, &AX); -++ DST = AX; -+ CONT; -+ ALU_MOD_X: -+- if (unlikely(SRC == 0)) -+- return 0; -+- tmp = (u32) DST; -+- DST = do_div(tmp, (u32) SRC); -++ AX = (u32) DST; -++ DST = do_div(AX, (u32) SRC); -+ CONT; -+ ALU64_MOD_K: -+- div64_u64_rem(DST, IMM, &tmp); -+- DST = tmp; -++ div64_u64_rem(DST, IMM, &AX); -++ DST = AX; -+ CONT; -+ ALU_MOD_K: -+- tmp = (u32) DST; -+- DST = do_div(tmp, (u32) IMM); -++ AX = (u32) DST; -++ DST = do_div(AX, (u32) IMM); -+ CONT; -+ ALU64_DIV_X: -+- if (unlikely(SRC == 0)) -+- return 0; -+ DST = div64_u64(DST, SRC); -+ CONT; -+ ALU_DIV_X: -+- if (unlikely(SRC == 0)) -+- return 0; -+- tmp = (u32) DST; -+- do_div(tmp, (u32) SRC); -+- DST = (u32) tmp; -++ AX = (u32) DST; -++ do_div(AX, (u32) SRC); -++ DST = (u32) AX; -+ CONT; -+ ALU64_DIV_K: -+ DST = div64_u64(DST, IMM); -+ CONT; -+ ALU_DIV_K: -+- tmp = (u32) DST; -+- do_div(tmp, (u32) IMM); -+- DST = (u32) tmp; -++ AX = (u32) DST; -++ do_div(AX, (u32) IMM); -++ DST = (u32) AX; -+ CONT; -+ ALU_END_TO_BE: -+ switch (IMM) { -+@@ -442,22 +1473,28 @@ select_insn: -+ BPF_R4, BPF_R5); -+ CONT; -+ -++ JMP_CALL_ARGS: -++ BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, -++ BPF_R3, BPF_R4, -++ BPF_R5, -++ insn + insn->off + 1); -++ CONT; -++ -+ JMP_TAIL_CALL: { -+ struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; -+ struct bpf_array *array = container_of(map, struct bpf_array, map); -+ struct bpf_prog *prog; -+- u64 index = BPF_R3; -++ u32 index = BPF_R3; -+ -+ if (unlikely(index >= array->map.max_entries)) -+ goto out; -+- -+ if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT)) -+ goto out; -+ -+ tail_call_cnt++; -+ -+ prog = READ_ONCE(array->ptrs[index]); -+- if (unlikely(!prog)) -++ if (!prog) -+ goto out; -+ -+ /* ARG1 at this point is guaranteed to point to CTX from -+@@ -470,97 +1507,49 @@ select_insn: -+ out: -+ CONT; -+ } -+- /* JMP */ -+ JMP_JA: -+ insn += insn->off; -+ CONT; -+- JMP_JEQ_X: -+- if (DST == SRC) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JEQ_K: -+- if (DST == IMM) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JNE_X: -+- if (DST != SRC) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JNE_K: -+- if (DST != IMM) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JGT_X: -+- if (DST > SRC) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JGT_K: -+- if (DST > IMM) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JGE_X: -+- if (DST >= SRC) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JGE_K: -+- if (DST >= IMM) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSGT_X: -+- if (((s64) DST) > ((s64) SRC)) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSGT_K: -+- if (((s64) DST) > ((s64) IMM)) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSGE_X: -+- if (((s64) DST) >= ((s64) SRC)) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSGE_K: -+- if (((s64) DST) >= ((s64) IMM)) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSET_X: -+- if (DST & SRC) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+- JMP_JSET_K: -+- if (DST & IMM) { -+- insn += insn->off; -+- CONT_JMP; -+- } -+- CONT; -+ JMP_EXIT: -+ return BPF_R0; -+- -++ /* JMP */ -++#define COND_JMP(SIGN, OPCODE, CMP_OP) \ -++ JMP_##OPCODE##_X: \ -++ if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ -++ insn += insn->off; \ -++ CONT_JMP; \ -++ } \ -++ CONT; \ -++ JMP32_##OPCODE##_X: \ -++ if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ -++ insn += insn->off; \ -++ CONT_JMP; \ -++ } \ -++ CONT; \ -++ JMP_##OPCODE##_K: \ -++ if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ -++ insn += insn->off; \ -++ CONT_JMP; \ -++ } \ -++ CONT; \ -++ JMP32_##OPCODE##_K: \ -++ if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ -++ insn += insn->off; \ -++ CONT_JMP; \ -++ } \ -++ CONT; -++ COND_JMP(u, JEQ, ==) -++ COND_JMP(u, JNE, !=) -++ COND_JMP(u, JGT, >) -++ COND_JMP(u, JLT, <) -++ COND_JMP(u, JGE, >=) -++ COND_JMP(u, JLE, <=) -++ COND_JMP(u, JSET, &) -++ COND_JMP(s, JSGT, >) -++ COND_JMP(s, JSLT, <) -++ COND_JMP(s, JSGE, >=) -++ COND_JMP(s, JSLE, <=) -++#undef COND_JMP -+ /* STX and ST and LDX*/ -+ #define LDST(SIZEOP, SIZE) \ -+ STX_MEM_##SIZEOP: \ -+@@ -586,77 +1575,108 @@ out: -+ atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) -+ (DST + insn->off)); -+ CONT; -+- LD_ABS_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */ -+- off = IMM; -+-load_word: -+- /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are -+- * only appearing in the programs where ctx == -+- * skb. All programs keep 'ctx' in regs[BPF_REG_CTX] -+- * == BPF_R6, bpf_convert_filter() saves it in BPF_R6, -+- * internal BPF verifier will check that BPF_R6 == -+- * ctx. -+- * -+- * BPF_ABS and BPF_IND are wrappers of function calls, -+- * so they scratch BPF_R1-BPF_R5 registers, preserve -+- * BPF_R6-BPF_R9, and store return value into BPF_R0. -+- * -+- * Implicit input: -+- * ctx == skb == BPF_R6 == CTX -+- * -+- * Explicit input: -+- * SRC == any register -+- * IMM == 32-bit immediate -++ -++ default_label: -++ /* If we ever reach this, we have a bug somewhere. Die hard here -++ * instead of just returning 0; we could be somewhere in a subprog, -++ * so execution could continue otherwise which we do /not/ want. -+ * -+- * Output: -+- * BPF_R0 - 8/16/32-bit skb data converted to cpu endianness -++ * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). -+ */ -++ pr_warn("BPF interpreter: unknown opcode %02x\n", insn->code); -++ BUG_ON(1); -++ return 0; -++} -+ -+- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 4, &tmp); -+- if (likely(ptr != NULL)) { -+- BPF_R0 = get_unaligned_be32(ptr); -+- CONT; -+- } -++#define PROG_NAME(stack_size) __bpf_prog_run##stack_size -++#define DEFINE_BPF_PROG_RUN(stack_size) \ -++static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ -++{ \ -++ u64 stack[stack_size / sizeof(u64)]; \ -++ u64 regs[MAX_BPF_EXT_REG]; \ -++\ -++ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ -++ ARG1 = (u64) (unsigned long) ctx; \ -++ return ___bpf_prog_run(regs, insn, stack); \ -++} -+ -+- return 0; -+- LD_ABS_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + imm32)) */ -+- off = IMM; -+-load_half: -+- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 2, &tmp); -+- if (likely(ptr != NULL)) { -+- BPF_R0 = get_unaligned_be16(ptr); -+- CONT; -+- } -++#define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size -++#define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ -++static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ -++ const struct bpf_insn *insn) \ -++{ \ -++ u64 stack[stack_size / sizeof(u64)]; \ -++ u64 regs[MAX_BPF_EXT_REG]; \ -++\ -++ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ -++ BPF_R1 = r1; \ -++ BPF_R2 = r2; \ -++ BPF_R3 = r3; \ -++ BPF_R4 = r4; \ -++ BPF_R5 = r5; \ -++ return ___bpf_prog_run(regs, insn, stack); \ -++} -+ -+- return 0; -+- LD_ABS_B: /* BPF_R0 = *(u8 *) (skb->data + imm32) */ -+- off = IMM; -+-load_byte: -+- ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 1, &tmp); -+- if (likely(ptr != NULL)) { -+- BPF_R0 = *(u8 *)ptr; -+- CONT; -+- } -++#define EVAL1(FN, X) FN(X) -++#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) -++#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) -++#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) -++#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) -++#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) -++ -++EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); -++EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); -++EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); -++ -++EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); -++EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); -++EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); -++ -++#define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), -++ -++static unsigned int (*interpreters[])(const void *ctx, -++ const struct bpf_insn *insn) = { -++EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) -++EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) -++EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) -++}; -++#undef PROG_NAME_LIST -++#define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), -++static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, -++ const struct bpf_insn *insn) = { -++EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) -++EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) -++EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) -++}; -++#undef PROG_NAME_LIST -+ -+- return 0; -+- LD_IND_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + src_reg + imm32)) */ -+- off = IMM + SRC; -+- goto load_word; -+- LD_IND_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + src_reg + imm32)) */ -+- off = IMM + SRC; -+- goto load_half; -+- LD_IND_B: /* BPF_R0 = *(u8 *) (skb->data + src_reg + imm32) */ -+- off = IMM + SRC; -+- goto load_byte; -++void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) -++{ -++ stack_depth = max_t(u32, stack_depth, 1); -++ insn->off = (s16) insn->imm; -++ insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - -++ __bpf_call_base_args; -++ insn->code = BPF_JMP | BPF_CALL_ARGS; -++} -+ -+- default_label: -+- /* If we ever reach this, we have a bug somewhere. */ -+- WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code); -+- return 0; -++#else -++static unsigned int __bpf_prog_ret0_warn(const void *ctx, -++ const struct bpf_insn *insn) -++{ -++ /* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON -++ * is not working properly, so warn about it! -++ */ -++ WARN_ON_ONCE(1); -++ return 0; -+ } -++#endif -+ -+ bool bpf_prog_array_compatible(struct bpf_array *array, -+ const struct bpf_prog *fp) -+ { -++ if (fp->kprobe_override) -++ return false; -++ -+ if (!array->owner_prog_type) { -+ /* There's no owner yet where we could check for -+ * compatibility. -+@@ -691,18 +1711,62 @@ static int bpf_check_tail_call(const str -+ return 0; -+ } -+ -++static void bpf_prog_select_func(struct bpf_prog *fp) -++{ -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++ u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); -++ -++ fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; -++#else -++ fp->bpf_func = __bpf_prog_ret0_warn; -++#endif -++} -++ -+ /** -+ * bpf_prog_select_runtime - select exec runtime for BPF program -+ * @fp: bpf_prog populated with internal BPF program -++ * @err: pointer to error variable -+ * -+ * Try to JIT eBPF program, if JIT is not available, use interpreter. -+ * The BPF program will be executed via BPF_PROG_RUN() macro. -+ */ -+-int bpf_prog_select_runtime(struct bpf_prog *fp) -++struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) -+ { -+- fp->bpf_func = (void *) __bpf_prog_run; -++ /* In case of BPF to BPF calls, verifier did all the prep -++ * work with regards to JITing, etc. -++ */ -++ if (fp->bpf_func) -++ goto finalize; -++ -++ bpf_prog_select_func(fp); -++ -++ /* eBPF JITs can rewrite the program in case constant -++ * blinding is active. However, in case of error during -++ * blinding, bpf_int_jit_compile() must always return a -++ * valid program, which in this case would simply not -++ * be JITed, but falls back to the interpreter. -++ */ -++ if (!bpf_prog_is_dev_bound(fp->aux)) { -++ *err = bpf_prog_alloc_jited_linfo(fp); -++ if (*err) -++ return fp; -++ -++ fp = bpf_int_jit_compile(fp); -++ if (!fp->jited) { -++ bpf_prog_free_jited_linfo(fp); -++#ifdef CONFIG_BPF_JIT_ALWAYS_ON -++ *err = -ENOTSUPP; -++ return fp; -++#endif -++ } else { -++ bpf_prog_free_unused_jited_linfo(fp); -++ } -++ } else { -++ *err = -EINVAL; -++ return fp; -++ } -+ -+- bpf_int_jit_compile(fp); -++finalize: -+ bpf_prog_lock_ro(fp); -+ -+ /* The tail call compatibility check can only be done at -+@@ -710,16 +1774,238 @@ int bpf_prog_select_runtime(struct bpf_p -+ * with JITed or non JITed program concatenations and not -+ * all eBPF JITs might immediately support all features. -+ */ -+- return bpf_check_tail_call(fp); -++ *err = bpf_check_tail_call(fp); -++ -++ return fp; -+ } -+ EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); -+ -++static unsigned int __bpf_prog_ret1(const void *ctx, -++ const struct bpf_insn *insn) -++{ -++ return 1; -++} -++ -++static struct bpf_prog_dummy { -++ struct bpf_prog prog; -++} dummy_bpf_prog = { -++ .prog = { -++ .bpf_func = __bpf_prog_ret1, -++ }, -++}; -++ -++/* to avoid allocating empty bpf_prog_array for cgroups that -++ * don't have bpf program attached use one global 'empty_prog_array' -++ * It will not be modified the caller of bpf_prog_array_alloc() -++ * (since caller requested prog_cnt == 0) -++ * that pointer should be 'freed' by bpf_prog_array_free() -++ */ -++static struct { -++ struct bpf_prog_array hdr; -++ struct bpf_prog *null_prog; -++} empty_prog_array = { -++ .null_prog = NULL, -++}; -++ -++struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) -++{ -++ if (prog_cnt) -++ return kzalloc(sizeof(struct bpf_prog_array) + -++ sizeof(struct bpf_prog_array_item) * -++ (prog_cnt + 1), -++ flags); -++ -++ return &empty_prog_array.hdr; -++} -++ -++void bpf_prog_array_free(struct bpf_prog_array *progs) -++{ -++ if (!progs || progs == &empty_prog_array.hdr) -++ return; -++ kfree_rcu(progs, rcu); -++} -++ -++int bpf_prog_array_length(struct bpf_prog_array *array) -++{ -++ struct bpf_prog_array_item *item; -++ u32 cnt = 0; -++ -++ for (item = array->items; item->prog; item++) -++ if (item->prog != &dummy_bpf_prog.prog) -++ cnt++; -++ return cnt; -++} -++ -++bool bpf_prog_array_is_empty(struct bpf_prog_array *array) -++{ -++ struct bpf_prog_array_item *item; -++ -++ for (item = array->items; item->prog; item++) -++ if (item->prog != &dummy_bpf_prog.prog) -++ return false; -++ return true; -++} -++ -++static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, -++ u32 *prog_ids, -++ u32 request_cnt) -++{ -++ struct bpf_prog_array_item *item; -++ int i = 0; -++ -++ for (item = array->items; item->prog; item++) { -++ if (item->prog == &dummy_bpf_prog.prog) -++ continue; -++ prog_ids[i] = item->prog->aux->id; -++ if (++i == request_cnt) { -++ item++; -++ break; -++ } -++ } -++ -++ return !!(item->prog); -++} -++ -++int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, -++ __u32 __user *prog_ids, u32 cnt) -++{ -++ unsigned long err = 0; -++ bool nospc; -++ u32 *ids; -++ -++ /* users of this function are doing: -++ * cnt = bpf_prog_array_length(); -++ * if (cnt > 0) -++ * bpf_prog_array_copy_to_user(..., cnt); -++ * so below kcalloc doesn't need extra cnt > 0 check. -++ */ -++ ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); -++ if (!ids) -++ return -ENOMEM; -++ nospc = bpf_prog_array_copy_core(array, ids, cnt); -++ err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); -++ kfree(ids); -++ if (err) -++ return -EFAULT; -++ if (nospc) -++ return -ENOSPC; -++ return 0; -++} -++ -++void bpf_prog_array_delete_safe(struct bpf_prog_array *array, -++ struct bpf_prog *old_prog) -++{ -++ struct bpf_prog_array_item *item; -++ -++ for (item = array->items; item->prog; item++) -++ if (item->prog == old_prog) { -++ WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); -++ break; -++ } -++} -++ -++int bpf_prog_array_copy(struct bpf_prog_array *old_array, -++ struct bpf_prog *exclude_prog, -++ struct bpf_prog *include_prog, -++ struct bpf_prog_array **new_array) -++{ -++ int new_prog_cnt, carry_prog_cnt = 0; -++ struct bpf_prog_array_item *existing; -++ struct bpf_prog_array *array; -++ bool found_exclude = false; -++ int new_prog_idx = 0; -++ -++ /* Figure out how many existing progs we need to carry over to -++ * the new array. -++ */ -++ if (old_array) { -++ existing = old_array->items; -++ for (; existing->prog; existing++) { -++ if (existing->prog == exclude_prog) { -++ found_exclude = true; -++ continue; -++ } -++ if (existing->prog != &dummy_bpf_prog.prog) -++ carry_prog_cnt++; -++ if (existing->prog == include_prog) -++ return -EEXIST; -++ } -++ } -++ -++ if (exclude_prog && !found_exclude) -++ return -ENOENT; -++ -++ /* How many progs (not NULL) will be in the new array? */ -++ new_prog_cnt = carry_prog_cnt; -++ if (include_prog) -++ new_prog_cnt += 1; -++ -++ /* Do we have any prog (not NULL) in the new array? */ -++ if (!new_prog_cnt) { -++ *new_array = NULL; -++ return 0; -++ } -++ -++ /* +1 as the end of prog_array is marked with NULL */ -++ array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); -++ if (!array) -++ return -ENOMEM; -++ -++ /* Fill in the new prog array */ -++ if (carry_prog_cnt) { -++ existing = old_array->items; -++ for (; existing->prog; existing++) -++ if (existing->prog != exclude_prog && -++ existing->prog != &dummy_bpf_prog.prog) { -++ array->items[new_prog_idx++].prog = -++ existing->prog; -++ } -++ } -++ if (include_prog) -++ array->items[new_prog_idx++].prog = include_prog; -++ array->items[new_prog_idx].prog = NULL; -++ *new_array = array; -++ return 0; -++} -++ -++int bpf_prog_array_copy_info(struct bpf_prog_array *array, -++ u32 *prog_ids, u32 request_cnt, -++ u32 *prog_cnt) -++{ -++ u32 cnt = 0; -++ -++ if (array) -++ cnt = bpf_prog_array_length(array); -++ -++ *prog_cnt = cnt; -++ -++ /* return early if user requested only program count or nothing to copy */ -++ if (!request_cnt || !cnt) -++ return 0; -++ -++ /* this function is called under trace/bpf_trace.c: bpf_event_mutex */ -++ return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC -++ : 0; -++} -++ -+ static void bpf_prog_free_deferred(struct work_struct *work) -+ { -+ struct bpf_prog_aux *aux; -++ int i; -+ -+ aux = container_of(work, struct bpf_prog_aux, work); -+- bpf_jit_free(aux->prog); -++#ifdef CONFIG_PERF_EVENTS -++ if (aux->prog->has_callchain_buf) -++ put_callchain_buffers(); -++#endif -++ for (i = 0; i < aux->func_cnt; i++) -++ bpf_jit_free(aux->func[i]); -++ if (aux->func_cnt) { -++ kfree(aux->func); -++ bpf_prog_unlock_free(aux->prog); -++ } else { -++ bpf_jit_free(aux->prog); -++ } -+ } -+ -+ /* Free internal BPF program */ -+@@ -740,7 +2026,7 @@ void bpf_user_rnd_init_once(void) -+ prandom_init_once(&bpf_user_rnd_state); -+ } -+ -+-u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_0(bpf_user_rnd_u32) -+ { -+ /* Should someone ever have the rather unwise idea to use some -+ * of the registers passed into this function, then note that -+@@ -753,7 +2039,7 @@ u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 -+ -+ state = &get_cpu_var(bpf_user_rnd_state); -+ res = prandom_u32_state(state); -+- put_cpu_var(state); -++ put_cpu_var(bpf_user_rnd_state); -+ -+ return res; -+ } -+@@ -762,18 +2048,36 @@ u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 -+ const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; -+ const struct bpf_func_proto bpf_map_update_elem_proto __weak; -+ const struct bpf_func_proto bpf_map_delete_elem_proto __weak; -++const struct bpf_func_proto bpf_map_push_elem_proto __weak; -++const struct bpf_func_proto bpf_map_pop_elem_proto __weak; -++const struct bpf_func_proto bpf_map_peek_elem_proto __weak; -++const struct bpf_func_proto bpf_spin_lock_proto __weak; -++const struct bpf_func_proto bpf_spin_unlock_proto __weak; -+ -+ const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; -+ const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; -++const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; -+ const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; -++ -+ const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; -+ const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; -+ const struct bpf_func_proto bpf_get_current_comm_proto __weak; -++const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; -++const struct bpf_func_proto bpf_get_local_storage_proto __weak; -++ -+ const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) -+ { -+ return NULL; -+ } -+ -++u64 __weak -++bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, -++ void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) -++{ -++ return -ENOTSUPP; -++} -++EXPORT_SYMBOL_GPL(bpf_event_output); -++ -+ /* Always built-in helper functions. */ -+ const struct bpf_func_proto bpf_tail_call_proto = { -+ .func = NULL, -+@@ -784,9 +2088,34 @@ const struct bpf_func_proto bpf_tail_cal -+ .arg3_type = ARG_ANYTHING, -+ }; -+ -+-/* For classic BPF JITs that don't implement bpf_int_jit_compile(). */ -+-void __weak bpf_int_jit_compile(struct bpf_prog *prog) -++/* Stub for JITs that only support cBPF. eBPF programs are interpreted. -++ * It is encouraged to implement bpf_int_jit_compile() instead, so that -++ * eBPF and implicitly also cBPF can get JITed! -++ */ -++struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) -+ { -++ return prog; -++} -++ -++/* Stub for JITs that support eBPF. All cBPF code gets transformed into -++ * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). -++ */ -++void __weak bpf_jit_compile(struct bpf_prog *prog) -++{ -++} -++ -++bool __weak bpf_helper_changes_pkt_data(void *func) -++{ -++ return false; -++} -++ -++/* Return TRUE if the JIT backend wants verifier to enable sub-register usage -++ * analysis code and wants explicit zero extension inserted by verifier. -++ * Otherwise, return FALSE. -++ */ -++bool __weak bpf_jit_needs_zext(void) -++{ -++ return false; -+ } -+ -+ /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call -+@@ -797,3 +2126,13 @@ int __weak skb_copy_bits(const struct sk -+ { -+ return -EFAULT; -+ } -++ -++DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); -++EXPORT_SYMBOL(bpf_stats_enabled_key); -++ -++/* All definitions of tracepoints related to BPF. */ -++#define CREATE_TRACE_POINTS -++#include -++ -++EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); -++EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); -+--- /dev/null -++++ b/kernel/bpf/devmap.c -+@@ -0,0 +1,698 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2017 Covalent IO, Inc. http://covalent.io -++ */ -++ -++/* Devmaps primary use is as a backend map for XDP BPF helper call -++ * bpf_redirect_map(). Because XDP is mostly concerned with performance we -++ * spent some effort to ensure the datapath with redirect maps does not use -++ * any locking. This is a quick note on the details. -++ * -++ * We have three possible paths to get into the devmap control plane bpf -++ * syscalls, bpf programs, and driver side xmit/flush operations. A bpf syscall -++ * will invoke an update, delete, or lookup operation. To ensure updates and -++ * deletes appear atomic from the datapath side xchg() is used to modify the -++ * netdev_map array. Then because the datapath does a lookup into the netdev_map -++ * array (read-only) from an RCU critical section we use call_rcu() to wait for -++ * an rcu grace period before free'ing the old data structures. This ensures the -++ * datapath always has a valid copy. However, the datapath does a "flush" -++ * operation that pushes any pending packets in the driver outside the RCU -++ * critical section. Each bpf_dtab_netdev tracks these pending operations using -++ * a per-cpu flush list. The bpf_dtab_netdev object will not be destroyed until -++ * this list is empty, indicating outstanding flush operations have completed. -++ * -++ * BPF syscalls may race with BPF program calls on any of the update, delete -++ * or lookup operations. As noted above the xchg() operation also keep the -++ * netdev_map consistent in this case. From the devmap side BPF programs -++ * calling into these operations are the same as multiple user space threads -++ * making system calls. -++ * -++ * Finally, any of the above may race with a netdev_unregister notifier. The -++ * unregister notifier must search for net devices in the map structure that -++ * contain a reference to the net device and remove them. This is a two step -++ * process (a) dereference the bpf_dtab_netdev object in netdev_map and (b) -++ * check to see if the ifindex is the same as the net_device being removed. -++ * When removing the dev a cmpxchg() is used to ensure the correct dev is -++ * removed, in the case of a concurrent update or delete operation it is -++ * possible that the initially referenced dev is no longer in the map. As the -++ * notifier hook walks the map we know that new dev references can not be -++ * added by the user because core infrastructure ensures dev_get_by_index() -++ * calls will fail at this point. -++ * -++ * The devmap_hash type is a map type which interprets keys as ifindexes and -++ * indexes these using a hashmap. This allows maps that use ifindex as key to be -++ * densely packed instead of having holes in the lookup array for unused -++ * ifindexes. The setup and packet enqueue/send code is shared between the two -++ * types of devmap; only the lookup and insertion is different. -++ */ -++#include -++#include -++#include -++#include -++ -++#define DEV_CREATE_FLAG_MASK \ -++ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY) -++ -++#define DEV_MAP_BULK_SIZE 16 -++struct bpf_dtab_netdev; -++ -++struct xdp_bulk_queue { -++ struct xdp_frame *q[DEV_MAP_BULK_SIZE]; -++ struct list_head flush_node; -++ struct net_device *dev_rx; -++ struct bpf_dtab_netdev *obj; -++ unsigned int count; -++}; -++ -++struct bpf_dtab_netdev { -++ struct net_device *dev; /* must be first member, due to tracepoint */ -++ struct hlist_node index_hlist; -++ struct bpf_dtab *dtab; -++ struct xdp_bulk_queue __percpu *bulkq; -++ struct rcu_head rcu; -++ unsigned int idx; /* keep track of map index for tracepoint */ -++}; -++ -++struct bpf_dtab { -++ struct bpf_map map; -++ struct bpf_dtab_netdev **netdev_map; /* DEVMAP type only */ -++ struct list_head __percpu *flush_list; -++ struct list_head list; -++ -++ /* these are only used for DEVMAP_HASH type maps */ -++ struct hlist_head *dev_index_head; -++ spinlock_t index_lock; -++ unsigned int items; -++ u32 n_buckets; -++}; -++ -++static DEFINE_SPINLOCK(dev_map_lock); -++static LIST_HEAD(dev_map_list); -++ -++static struct hlist_head *dev_map_create_hash(unsigned int entries, -++ int numa_node) -++{ -++ int i; -++ struct hlist_head *hash; -++ -++ hash = bpf_map_area_alloc(entries * sizeof(*hash), numa_node); -++ if (hash != NULL) -++ for (i = 0; i < entries; i++) -++ INIT_HLIST_HEAD(&hash[i]); -++ -++ return hash; -++} -++ -++static inline struct hlist_head *dev_map_index_hash(struct bpf_dtab *dtab, -++ int idx) -++{ -++ return &dtab->dev_index_head[idx & (dtab->n_buckets - 1)]; -++} -++ -++static int dev_map_init_map(struct bpf_dtab *dtab, union bpf_attr *attr) -++{ -++ int err, cpu; -++ u64 cost; -++ -++ /* check sanity of attributes */ -++ if (attr->max_entries == 0 || attr->key_size != 4 || -++ attr->value_size != 4 || attr->map_flags & ~DEV_CREATE_FLAG_MASK) -++ return -EINVAL; -++ -++ /* Lookup returns a pointer straight to dev->ifindex, so make sure the -++ * verifier prevents writes from the BPF side -++ */ -++ attr->map_flags |= BPF_F_RDONLY_PROG; -++ -++ -++ bpf_map_init_from_attr(&dtab->map, attr); -++ -++ /* make sure page count doesn't overflow */ -++ cost = (u64) sizeof(struct list_head) * num_possible_cpus(); -++ -++ if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { -++ dtab->n_buckets = roundup_pow_of_two(dtab->map.max_entries); -++ -++ if (!dtab->n_buckets) /* Overflow check */ -++ return -EINVAL; -++ cost += (u64) sizeof(struct hlist_head) * dtab->n_buckets; -++ } else { -++ cost += (u64) dtab->map.max_entries * sizeof(struct bpf_dtab_netdev *); -++ } -++ -++ /* if map size is larger than memlock limit, reject it */ -++ err = bpf_map_charge_init(&dtab->map.memory, cost); -++ if (err) -++ return -EINVAL; -++ -++ dtab->flush_list = alloc_percpu(struct list_head); -++ if (!dtab->flush_list) -++ goto free_charge; -++ -++ for_each_possible_cpu(cpu) -++ INIT_LIST_HEAD(per_cpu_ptr(dtab->flush_list, cpu)); -++ -++ if (attr->map_type == BPF_MAP_TYPE_DEVMAP_HASH) { -++ dtab->dev_index_head = dev_map_create_hash(dtab->n_buckets, -++ dtab->map.numa_node); -++ if (!dtab->dev_index_head) -++ goto free_percpu; -++ -++ spin_lock_init(&dtab->index_lock); -++ } else { -++ dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries * -++ sizeof(struct bpf_dtab_netdev *), -++ dtab->map.numa_node); -++ if (!dtab->netdev_map) -++ goto free_percpu; -++ } -++ -++ return 0; -++ -++free_percpu: -++ free_percpu(dtab->flush_list); -++free_charge: -++ bpf_map_charge_finish(&dtab->map.memory); -++ return -ENOMEM; -++} -++ -++static struct bpf_map *dev_map_alloc(union bpf_attr *attr) -++{ -++ struct bpf_dtab *dtab; -++ int err; -++ -++ if (!capable(CAP_NET_ADMIN)) -++ return ERR_PTR(-EPERM); -++ -++ dtab = kzalloc(sizeof(*dtab), GFP_USER); -++ if (!dtab) -++ return ERR_PTR(-ENOMEM); -++ -++ err = dev_map_init_map(dtab, attr); -++ if (err) { -++ kfree(dtab); -++ return ERR_PTR(err); -++ } -++ -++ spin_lock(&dev_map_lock); -++ list_add_tail_rcu(&dtab->list, &dev_map_list); -++ spin_unlock(&dev_map_lock); -++ -++ return &dtab->map; -++} -++ -++static void dev_map_free(struct bpf_map *map) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ int i, cpu; -++ -++ /* At this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, -++ * so the programs (can be more than one that used this map) were -++ * disconnected from events. Wait for outstanding critical sections in -++ * these programs to complete. The rcu critical section only guarantees -++ * no further reads against netdev_map. It does __not__ ensure pending -++ * flush operations (if any) are complete. -++ */ -++ -++ spin_lock(&dev_map_lock); -++ list_del_rcu(&dtab->list); -++ spin_unlock(&dev_map_lock); -++ -++ bpf_clear_redirect_map(map); -++ synchronize_rcu(); -++ -++ /* Make sure prior __dev_map_entry_free() have completed. */ -++ rcu_barrier(); -++ -++ /* To ensure all pending flush operations have completed wait for flush -++ * list to empty on _all_ cpus. -++ * Because the above synchronize_rcu() ensures the map is disconnected -++ * from the program we can assume no new items will be added. -++ */ -++ for_each_online_cpu(cpu) { -++ struct list_head *flush_list = per_cpu_ptr(dtab->flush_list, cpu); -++ -++ while (!list_empty(flush_list)) -++ cond_resched(); -++ } -++ -++ if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { -++ for (i = 0; i < dtab->n_buckets; i++) { -++ struct bpf_dtab_netdev *dev; -++ struct hlist_head *head; -++ struct hlist_node *next; -++ -++ head = dev_map_index_hash(dtab, i); -++ -++ hlist_for_each_entry_safe(dev, next, head, index_hlist) { -++ hlist_del_rcu(&dev->index_hlist); -++ free_percpu(dev->bulkq); -++ dev_put(dev->dev); -++ kfree(dev); -++ } -++ } -++ -++ bpf_map_area_free(dtab->dev_index_head); -++ } else { -++ for (i = 0; i < dtab->map.max_entries; i++) { -++ struct bpf_dtab_netdev *dev; -++ -++ dev = dtab->netdev_map[i]; -++ if (!dev) -++ continue; -++ -++ free_percpu(dev->bulkq); -++ dev_put(dev->dev); -++ kfree(dev); -++ } -++ -++ bpf_map_area_free(dtab->netdev_map); -++ } -++ -++ free_percpu(dtab->flush_list); -++ kfree(dtab); -++} -++ -++static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ u32 index = key ? *(u32 *)key : U32_MAX; -++ u32 *next = next_key; -++ -++ if (index >= dtab->map.max_entries) { -++ *next = 0; -++ return 0; -++ } -++ -++ if (index == dtab->map.max_entries - 1) -++ return -ENOENT; -++ *next = index + 1; -++ return 0; -++} -++ -++struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct hlist_head *head = dev_map_index_hash(dtab, key); -++ struct bpf_dtab_netdev *dev; -++ -++ hlist_for_each_entry_rcu(dev, head, index_hlist) -++ if (dev->idx == key) -++ return dev; -++ -++ return NULL; -++} -++ -++static int dev_map_hash_get_next_key(struct bpf_map *map, void *key, -++ void *next_key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ u32 idx, *next = next_key; -++ struct bpf_dtab_netdev *dev, *next_dev; -++ struct hlist_head *head; -++ int i = 0; -++ -++ if (!key) -++ goto find_first; -++ -++ idx = *(u32 *)key; -++ -++ dev = __dev_map_hash_lookup_elem(map, idx); -++ if (!dev) -++ goto find_first; -++ -++ next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&dev->index_hlist)), -++ struct bpf_dtab_netdev, index_hlist); -++ -++ if (next_dev) { -++ *next = next_dev->idx; -++ return 0; -++ } -++ -++ i = idx & (dtab->n_buckets - 1); -++ i++; -++ -++ find_first: -++ for (; i < dtab->n_buckets; i++) { -++ head = dev_map_index_hash(dtab, i); -++ -++ next_dev = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)), -++ struct bpf_dtab_netdev, -++ index_hlist); -++ if (next_dev) { -++ *next = next_dev->idx; -++ return 0; -++ } -++ } -++ -++ return -ENOENT; -++} -++ -++/* __dev_map_flush is called from xdp_do_flush_map() which _must_ be signaled -++ * from the driver before returning from its napi->poll() routine. The poll() -++ * routine is called either from busy_poll context or net_rx_action signaled -++ * from NET_RX_SOFTIRQ. Either way the poll routine must complete before the -++ * net device can be torn down. On devmap tear down we ensure the flush list -++ * is empty before completing to ensure all flush operations have completed. -++ */ -++void __dev_map_flush(struct bpf_map *map) -++{ -++} -++ -++/* rcu_read_lock (from syscall and BPF contexts) ensures that if a delete and/or -++ * update happens in parallel here a dev_put wont happen until after reading the -++ * ifindex. -++ */ -++struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct bpf_dtab_netdev *obj; -++ -++ if (key >= map->max_entries) -++ return NULL; -++ -++ obj = READ_ONCE(dtab->netdev_map[key]); -++ return obj; -++} -++ -++int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, -++ struct net_device *dev_rx) -++{ -++ return -EOPNOTSUPP; -++} -++ -++int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, -++ struct bpf_prog *xdp_prog) -++{ -++ return -EOPNOTSUPP; -++} -++ -++static void *dev_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_dtab_netdev *obj = __dev_map_lookup_elem(map, *(u32 *)key); -++ struct net_device *dev = obj ? obj->dev : NULL; -++ -++ return dev ? &dev->ifindex : NULL; -++} -++ -++static void *dev_map_hash_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_dtab_netdev *obj = __dev_map_hash_lookup_elem(map, -++ *(u32 *)key); -++ struct net_device *dev = obj ? obj->dev : NULL; -++ -++ return dev ? &dev->ifindex : NULL; -++} -++ -++static void __dev_map_entry_free(struct rcu_head *rcu) -++{ -++ struct bpf_dtab_netdev *dev; -++ -++ dev = container_of(rcu, struct bpf_dtab_netdev, rcu); -++ free_percpu(dev->bulkq); -++ dev_put(dev->dev); -++ kfree(dev); -++} -++ -++static int dev_map_delete_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct bpf_dtab_netdev *old_dev; -++ int k = *(u32 *)key; -++ -++ if (k >= map->max_entries) -++ return -EINVAL; -++ -++ /* Use call_rcu() here to ensure any rcu critical sections have -++ * completed, but this does not guarantee a flush has happened -++ * yet. Because driver side rcu_read_lock/unlock only protects the -++ * running XDP program. However, for pending flush operations the -++ * dev and ctx are stored in another per cpu map. And additionally, -++ * the driver tear down ensures all soft irqs are complete before -++ * removing the net device in the case of dev_put equals zero. -++ */ -++ old_dev = xchg(&dtab->netdev_map[k], NULL); -++ if (old_dev) -++ call_rcu(&old_dev->rcu, __dev_map_entry_free); -++ return 0; -++} -++ -++static int dev_map_hash_delete_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct bpf_dtab_netdev *old_dev; -++ int k = *(u32 *)key; -++ unsigned long flags; -++ int ret = -ENOENT; -++ -++ spin_lock_irqsave(&dtab->index_lock, flags); -++ -++ old_dev = __dev_map_hash_lookup_elem(map, k); -++ if (old_dev) { -++ dtab->items--; -++ hlist_del_init_rcu(&old_dev->index_hlist); -++ call_rcu(&old_dev->rcu, __dev_map_entry_free); -++ ret = 0; -++ } -++ spin_unlock_irqrestore(&dtab->index_lock, flags); -++ -++ return ret; -++} -++ -++static struct bpf_dtab_netdev *__dev_map_alloc_node(struct net *net, -++ struct bpf_dtab *dtab, -++ u32 ifindex, -++ unsigned int idx) -++{ -++ gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN; -++ struct bpf_dtab_netdev *dev; -++ struct xdp_bulk_queue *bq; -++ int cpu; -++ -++ dev = kmalloc_node(sizeof(*dev), gfp, dtab->map.numa_node); -++ if (!dev) -++ return ERR_PTR(-ENOMEM); -++ -++ dev->bulkq = __alloc_percpu_gfp(sizeof(*dev->bulkq), -++ sizeof(void *), gfp); -++ if (!dev->bulkq) { -++ kfree(dev); -++ return ERR_PTR(-ENOMEM); -++ } -++ -++ for_each_possible_cpu(cpu) { -++ bq = per_cpu_ptr(dev->bulkq, cpu); -++ bq->obj = dev; -++ } -++ -++ dev->dev = dev_get_by_index(net, ifindex); -++ if (!dev->dev) { -++ free_percpu(dev->bulkq); -++ kfree(dev); -++ return ERR_PTR(-EINVAL); -++ } -++ -++ dev->idx = idx; -++ dev->dtab = dtab; -++ -++ return dev; -++} -++ -++static int __dev_map_update_elem(struct net *net, struct bpf_map *map, -++ void *key, void *value, u64 map_flags) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct bpf_dtab_netdev *dev, *old_dev; -++ u32 ifindex = *(u32 *)value; -++ u32 i = *(u32 *)key; -++ -++ if (unlikely(map_flags > BPF_EXIST)) -++ return -EINVAL; -++ if (unlikely(i >= dtab->map.max_entries)) -++ return -E2BIG; -++ if (unlikely(map_flags == BPF_NOEXIST)) -++ return -EEXIST; -++ -++ if (!ifindex) { -++ dev = NULL; -++ } else { -++ dev = __dev_map_alloc_node(net, dtab, ifindex, i); -++ if (IS_ERR(dev)) -++ return PTR_ERR(dev); -++ } -++ -++ /* Use call_rcu() here to ensure rcu critical sections have completed -++ * Remembering the driver side flush operation will happen before the -++ * net device is removed. -++ */ -++ old_dev = xchg(&dtab->netdev_map[i], dev); -++ if (old_dev) -++ call_rcu(&old_dev->rcu, __dev_map_entry_free); -++ -++ return 0; -++} -++ -++static int dev_map_update_elem(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ return __dev_map_update_elem(current->nsproxy->net_ns, -++ map, key, value, map_flags); -++} -++ -++static int __dev_map_hash_update_elem(struct net *net, struct bpf_map *map, -++ void *key, void *value, u64 map_flags) -++{ -++ struct bpf_dtab *dtab = container_of(map, struct bpf_dtab, map); -++ struct bpf_dtab_netdev *dev, *old_dev; -++ u32 ifindex = *(u32 *)value; -++ u32 idx = *(u32 *)key; -++ unsigned long flags; -++ int err = -EEXIST; -++ -++ if (unlikely(map_flags > BPF_EXIST || !ifindex)) -++ return -EINVAL; -++ -++ spin_lock_irqsave(&dtab->index_lock, flags); -++ -++ old_dev = __dev_map_hash_lookup_elem(map, idx); -++ if (old_dev && (map_flags & BPF_NOEXIST)) -++ goto out_err; -++ -++ dev = __dev_map_alloc_node(net, dtab, ifindex, idx); -++ if (IS_ERR(dev)) { -++ err = PTR_ERR(dev); -++ goto out_err; -++ } -++ -++ if (old_dev) { -++ hlist_del_rcu(&old_dev->index_hlist); -++ } else { -++ if (dtab->items >= dtab->map.max_entries) { -++ spin_unlock_irqrestore(&dtab->index_lock, flags); -++ call_rcu(&dev->rcu, __dev_map_entry_free); -++ return -E2BIG; -++ } -++ dtab->items++; -++ } -++ -++ hlist_add_head_rcu(&dev->index_hlist, -++ dev_map_index_hash(dtab, idx)); -++ spin_unlock_irqrestore(&dtab->index_lock, flags); -++ -++ if (old_dev) -++ call_rcu(&old_dev->rcu, __dev_map_entry_free); -++ -++ return 0; -++ -++out_err: -++ spin_unlock_irqrestore(&dtab->index_lock, flags); -++ return err; -++} -++ -++static int dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ return __dev_map_hash_update_elem(current->nsproxy->net_ns, -++ map, key, value, map_flags); -++} -++ -++const struct bpf_map_ops dev_map_ops = { -++ .map_alloc = dev_map_alloc, -++ .map_free = dev_map_free, -++ .map_get_next_key = dev_map_get_next_key, -++ .map_lookup_elem = dev_map_lookup_elem, -++ .map_update_elem = dev_map_update_elem, -++ .map_delete_elem = dev_map_delete_elem, -++ .map_check_btf = map_check_no_btf, -++}; -++ -++const struct bpf_map_ops dev_map_hash_ops = { -++ .map_alloc = dev_map_alloc, -++ .map_free = dev_map_free, -++ .map_get_next_key = dev_map_hash_get_next_key, -++ .map_lookup_elem = dev_map_hash_lookup_elem, -++ .map_update_elem = dev_map_hash_update_elem, -++ .map_delete_elem = dev_map_hash_delete_elem, -++ .map_check_btf = map_check_no_btf, -++}; -++ -++static void dev_map_hash_remove_netdev(struct bpf_dtab *dtab, -++ struct net_device *netdev) -++{ -++ unsigned long flags; -++ u32 i; -++ -++ spin_lock_irqsave(&dtab->index_lock, flags); -++ for (i = 0; i < dtab->n_buckets; i++) { -++ struct bpf_dtab_netdev *dev; -++ struct hlist_head *head; -++ struct hlist_node *next; -++ -++ head = dev_map_index_hash(dtab, i); -++ -++ hlist_for_each_entry_safe(dev, next, head, index_hlist) { -++ if (netdev != dev->dev) -++ continue; -++ -++ dtab->items--; -++ hlist_del_rcu(&dev->index_hlist); -++ call_rcu(&dev->rcu, __dev_map_entry_free); -++ } -++ } -++ spin_unlock_irqrestore(&dtab->index_lock, flags); -++} -++ -++static int dev_map_notification(struct notifier_block *notifier, -++ ulong event, void *ptr) -++{ -++ struct net_device *netdev = netdev_notifier_info_to_dev(ptr); -++ struct bpf_dtab *dtab; -++ int i; -++ -++ switch (event) { -++ case NETDEV_UNREGISTER: -++ /* This rcu_read_lock/unlock pair is needed because -++ * dev_map_list is an RCU list AND to ensure a delete -++ * operation does not free a netdev_map entry while we -++ * are comparing it against the netdev being unregistered. -++ */ -++ rcu_read_lock(); -++ list_for_each_entry_rcu(dtab, &dev_map_list, list) { -++ if (dtab->map.map_type == BPF_MAP_TYPE_DEVMAP_HASH) { -++ dev_map_hash_remove_netdev(dtab, netdev); -++ continue; -++ } -++ -++ for (i = 0; i < dtab->map.max_entries; i++) { -++ struct bpf_dtab_netdev *dev, *odev; -++ -++ dev = READ_ONCE(dtab->netdev_map[i]); -++ if (!dev || netdev != dev->dev) -++ continue; -++ odev = cmpxchg(&dtab->netdev_map[i], dev, NULL); -++ if (dev == odev) -++ call_rcu(&dev->rcu, -++ __dev_map_entry_free); -++ } -++ } -++ rcu_read_unlock(); -++ break; -++ default: -++ break; -++ } -++ return NOTIFY_OK; -++} -++ -++static struct notifier_block dev_map_notifier = { -++ .notifier_call = dev_map_notification, -++}; -++ -++static int __init dev_map_init(void) -++{ -++ /* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */ -++ BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) != -++ offsetof(struct _bpf_dtab_netdev, dev)); -++ register_netdevice_notifier(&dev_map_notifier); -++ return 0; -++} -++ -++subsys_initcall(dev_map_init); -+--- /dev/null -++++ b/kernel/bpf/disasm.c -+@@ -0,0 +1,258 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -++ * Copyright (c) 2016 Facebook -++ */ -++ -++#include -++ -++#include "disasm.h" -++ -++#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x) -++static const char * const func_id_str[] = { -++ __BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN) -++}; -++#undef __BPF_FUNC_STR_FN -++ -++static const char *__func_get_name(const struct bpf_insn_cbs *cbs, -++ const struct bpf_insn *insn, -++ char *buff, size_t len) -++{ -++ BUILD_BUG_ON(ARRAY_SIZE(func_id_str) != __BPF_FUNC_MAX_ID); -++ -++ if (insn->src_reg != BPF_PSEUDO_CALL && -++ insn->imm >= 0 && insn->imm < __BPF_FUNC_MAX_ID && -++ func_id_str[insn->imm]) -++ return func_id_str[insn->imm]; -++ -++ if (cbs && cbs->cb_call) -++ return cbs->cb_call(cbs->private_data, insn); -++ -++ if (insn->src_reg == BPF_PSEUDO_CALL) -++ snprintf(buff, len, "%+d", insn->imm); -++ -++ return buff; -++} -++ -++static const char *__func_imm_name(const struct bpf_insn_cbs *cbs, -++ const struct bpf_insn *insn, -++ u64 full_imm, char *buff, size_t len) -++{ -++ if (cbs && cbs->cb_imm) -++ return cbs->cb_imm(cbs->private_data, insn, full_imm); -++ -++ snprintf(buff, len, "0x%llx", (unsigned long long)full_imm); -++ return buff; -++} -++ -++const char *func_id_name(int id) -++{ -++ if (id >= 0 && id < __BPF_FUNC_MAX_ID && func_id_str[id]) -++ return func_id_str[id]; -++ else -++ return "unknown"; -++} -++ -++const char *const bpf_class_string[8] = { -++ [BPF_LD] = "ld", -++ [BPF_LDX] = "ldx", -++ [BPF_ST] = "st", -++ [BPF_STX] = "stx", -++ [BPF_ALU] = "alu", -++ [BPF_JMP] = "jmp", -++ [BPF_JMP32] = "jmp32", -++ [BPF_ALU64] = "alu64", -++}; -++ -++const char *const bpf_alu_string[16] = { -++ [BPF_ADD >> 4] = "+=", -++ [BPF_SUB >> 4] = "-=", -++ [BPF_MUL >> 4] = "*=", -++ [BPF_DIV >> 4] = "/=", -++ [BPF_OR >> 4] = "|=", -++ [BPF_AND >> 4] = "&=", -++ [BPF_LSH >> 4] = "<<=", -++ [BPF_RSH >> 4] = ">>=", -++ [BPF_NEG >> 4] = "neg", -++ [BPF_MOD >> 4] = "%=", -++ [BPF_XOR >> 4] = "^=", -++ [BPF_MOV >> 4] = "=", -++ [BPF_ARSH >> 4] = "s>>=", -++ [BPF_END >> 4] = "endian", -++}; -++ -++static const char *const bpf_ldst_string[] = { -++ [BPF_W >> 3] = "u32", -++ [BPF_H >> 3] = "u16", -++ [BPF_B >> 3] = "u8", -++ [BPF_DW >> 3] = "u64", -++}; -++ -++static const char *const bpf_jmp_string[16] = { -++ [BPF_JA >> 4] = "jmp", -++ [BPF_JEQ >> 4] = "==", -++ [BPF_JGT >> 4] = ">", -++ [BPF_JLT >> 4] = "<", -++ [BPF_JGE >> 4] = ">=", -++ [BPF_JLE >> 4] = "<=", -++ [BPF_JSET >> 4] = "&", -++ [BPF_JNE >> 4] = "!=", -++ [BPF_JSGT >> 4] = "s>", -++ [BPF_JSLT >> 4] = "s<", -++ [BPF_JSGE >> 4] = "s>=", -++ [BPF_JSLE >> 4] = "s<=", -++ [BPF_CALL >> 4] = "call", -++ [BPF_EXIT >> 4] = "exit", -++}; -++ -++static void print_bpf_end_insn(bpf_insn_print_t verbose, -++ void *private_data, -++ const struct bpf_insn *insn) -++{ -++ verbose(private_data, "(%02x) r%d = %s%d r%d\n", -++ insn->code, insn->dst_reg, -++ BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le", -++ insn->imm, insn->dst_reg); -++} -++ -++void print_bpf_insn(const struct bpf_insn_cbs *cbs, -++ const struct bpf_insn *insn, -++ bool allow_ptr_leaks) -++{ -++ const bpf_insn_print_t verbose = cbs->cb_print; -++ u8 class = BPF_CLASS(insn->code); -++ -++ if (class == BPF_ALU || class == BPF_ALU64) { -++ if (BPF_OP(insn->code) == BPF_END) { -++ if (class == BPF_ALU64) -++ verbose(cbs->private_data, "BUG_alu64_%02x\n", insn->code); -++ else -++ print_bpf_end_insn(verbose, cbs->private_data, insn); -++ } else if (BPF_OP(insn->code) == BPF_NEG) { -++ verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n", -++ insn->code, class == BPF_ALU ? 'w' : 'r', -++ insn->dst_reg, class == BPF_ALU ? 'w' : 'r', -++ insn->dst_reg); -++ } else if (BPF_SRC(insn->code) == BPF_X) { -++ verbose(cbs->private_data, "(%02x) %c%d %s %c%d\n", -++ insn->code, class == BPF_ALU ? 'w' : 'r', -++ insn->dst_reg, -++ bpf_alu_string[BPF_OP(insn->code) >> 4], -++ class == BPF_ALU ? 'w' : 'r', -++ insn->src_reg); -++ } else { -++ verbose(cbs->private_data, "(%02x) %c%d %s %d\n", -++ insn->code, class == BPF_ALU ? 'w' : 'r', -++ insn->dst_reg, -++ bpf_alu_string[BPF_OP(insn->code) >> 4], -++ insn->imm); -++ } -++ } else if (class == BPF_STX) { -++ if (BPF_MODE(insn->code) == BPF_MEM) -++ verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n", -++ insn->code, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->dst_reg, -++ insn->off, insn->src_reg); -++ else if (BPF_MODE(insn->code) == BPF_XADD) -++ verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) += r%d\n", -++ insn->code, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->dst_reg, insn->off, -++ insn->src_reg); -++ else -++ verbose(cbs->private_data, "BUG_%02x\n", insn->code); -++ } else if (class == BPF_ST) { -++ if (BPF_MODE(insn->code) != BPF_MEM) { -++ verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); -++ return; -++ } -++ verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", -++ insn->code, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->dst_reg, -++ insn->off, insn->imm); -++ } else if (class == BPF_LDX) { -++ if (BPF_MODE(insn->code) != BPF_MEM) { -++ verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); -++ return; -++ } -++ verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n", -++ insn->code, insn->dst_reg, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->src_reg, insn->off); -++ } else if (class == BPF_LD) { -++ if (BPF_MODE(insn->code) == BPF_ABS) { -++ verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n", -++ insn->code, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->imm); -++ } else if (BPF_MODE(insn->code) == BPF_IND) { -++ verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n", -++ insn->code, -++ bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -++ insn->src_reg, insn->imm); -++ } else if (BPF_MODE(insn->code) == BPF_IMM && -++ BPF_SIZE(insn->code) == BPF_DW) { -++ /* At this point, we already made sure that the second -++ * part of the ldimm64 insn is accessible. -++ */ -++ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; -++ bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || -++ insn->src_reg == BPF_PSEUDO_MAP_VALUE; -++ char tmp[64]; -++ -++ if (is_ptr && !allow_ptr_leaks) -++ imm = 0; -++ -++ verbose(cbs->private_data, "(%02x) r%d = %s\n", -++ insn->code, insn->dst_reg, -++ __func_imm_name(cbs, insn, imm, -++ tmp, sizeof(tmp))); -++ } else { -++ verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code); -++ return; -++ } -++ } else if (class == BPF_JMP32 || class == BPF_JMP) { -++ u8 opcode = BPF_OP(insn->code); -++ -++ if (opcode == BPF_CALL) { -++ char tmp[64]; -++ -++ if (insn->src_reg == BPF_PSEUDO_CALL) { -++ verbose(cbs->private_data, "(%02x) call pc%s\n", -++ insn->code, -++ __func_get_name(cbs, insn, -++ tmp, sizeof(tmp))); -++ } else { -++ strcpy(tmp, "unknown"); -++ verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code, -++ __func_get_name(cbs, insn, -++ tmp, sizeof(tmp)), -++ insn->imm); -++ } -++ } else if (insn->code == (BPF_JMP | BPF_JA)) { -++ verbose(cbs->private_data, "(%02x) goto pc%+d\n", -++ insn->code, insn->off); -++ } else if (insn->code == (BPF_JMP | BPF_EXIT)) { -++ verbose(cbs->private_data, "(%02x) exit\n", insn->code); -++ } else if (BPF_SRC(insn->code) == BPF_X) { -++ verbose(cbs->private_data, -++ "(%02x) if %c%d %s %c%d goto pc%+d\n", -++ insn->code, class == BPF_JMP32 ? 'w' : 'r', -++ insn->dst_reg, -++ bpf_jmp_string[BPF_OP(insn->code) >> 4], -++ class == BPF_JMP32 ? 'w' : 'r', -++ insn->src_reg, insn->off); -++ } else { -++ verbose(cbs->private_data, -++ "(%02x) if %c%d %s 0x%x goto pc%+d\n", -++ insn->code, class == BPF_JMP32 ? 'w' : 'r', -++ insn->dst_reg, -++ bpf_jmp_string[BPF_OP(insn->code) >> 4], -++ insn->imm, insn->off); -++ } -++ } else { -++ verbose(cbs->private_data, "(%02x) %s\n", -++ insn->code, bpf_class_string[class]); -++ } -++} -+--- /dev/null -++++ b/kernel/bpf/disasm.h -+@@ -0,0 +1,40 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -++ * Copyright (c) 2016 Facebook -++ */ -++ -++#ifndef __BPF_DISASM_H__ -++#define __BPF_DISASM_H__ -++ -++#include -++#include -++#include -++#ifndef __KERNEL__ -++#include -++#include -++#endif -++ -++extern const char *const bpf_alu_string[16]; -++extern const char *const bpf_class_string[8]; -++ -++const char *func_id_name(int id); -++ -++typedef __printf(2, 3) void (*bpf_insn_print_t)(void *private_data, -++ const char *, ...); -++typedef const char *(*bpf_insn_revmap_call_t)(void *private_data, -++ const struct bpf_insn *insn); -++typedef const char *(*bpf_insn_print_imm_t)(void *private_data, -++ const struct bpf_insn *insn, -++ __u64 full_imm); -++ -++struct bpf_insn_cbs { -++ bpf_insn_print_t cb_print; -++ bpf_insn_revmap_call_t cb_call; -++ bpf_insn_print_imm_t cb_imm; -++ void *private_data; -++}; -++ -++void print_bpf_insn(const struct bpf_insn_cbs *cbs, -++ const struct bpf_insn *insn, -++ bool allow_ptr_leaks); -++#endif -+--- a/kernel/bpf/hashtab.c -++++ b/kernel/bpf/hashtab.c -+@@ -1,147 +1,467 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+- * -+- * This program is distributed in the hope that it will be useful, but -+- * WITHOUT ANY WARRANTY; without even the implied warranty of -+- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+- * General Public License for more details. -++ * Copyright (c) 2016 Facebook -+ */ -+ #include -++#include -+ #include -+ #include -+-#include -++#include -++#include -++#include -++#include "percpu_freelist.h" -++#include "bpf_lru_list.h" -++#include "map_in_map.h" -++ -++#define HTAB_CREATE_FLAG_MASK \ -++ (BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \ -++ BPF_F_ACCESS_MASK | BPF_F_ZERO_SEED) -++ -++struct bucket { -++ struct hlist_nulls_head head; -++ raw_spinlock_t lock; -++}; -+ -+ struct bpf_htab { -+ struct bpf_map map; -+- struct hlist_head *buckets; -+- raw_spinlock_t lock; -+- u32 count; /* number of elements in this hashtable */ -++ struct bucket *buckets; -++ void *elems; -++ union { -++ struct pcpu_freelist freelist; -++ struct bpf_lru lru; -++ }; -++ struct htab_elem *__percpu *extra_elems; -++ atomic_t count; /* number of elements in this hashtable */ -+ u32 n_buckets; /* number of hash buckets */ -+ u32 elem_size; /* size of each element in bytes */ -++ u32 hashrnd; -+ }; -+ -+ /* each htab element is struct htab_elem + key + value */ -+ struct htab_elem { -+- struct hlist_node hash_node; -+- struct rcu_head rcu; -++ union { -++ struct hlist_nulls_node hash_node; -++ struct { -++ void *padding; -++ union { -++ struct bpf_htab *htab; -++ struct pcpu_freelist_node fnode; -++ }; -++ }; -++ }; -++ union { -++ struct rcu_head rcu; -++ struct bpf_lru_node lru_node; -++ }; -+ u32 hash; -+ char key[0] __aligned(8); -+ }; -+ -++static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node); -++ -++static bool htab_is_lru(const struct bpf_htab *htab) -++{ -++ return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || -++ htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; -++} -++ -++static bool htab_is_percpu(const struct bpf_htab *htab) -++{ -++ return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; -++} -++ -++static bool htab_is_prealloc(const struct bpf_htab *htab) -++{ -++ return !(htab->map.map_flags & BPF_F_NO_PREALLOC); -++} -++ -++static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size, -++ void __percpu *pptr) -++{ -++ *(void __percpu **)(l->key + key_size) = pptr; -++} -++ -++static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size) -++{ -++ return *(void __percpu **)(l->key + key_size); -++} -++ -++static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) -++{ -++ return *(void **)(l->key + roundup(map->key_size, 8)); -++} -++ -++static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i) -++{ -++ return (struct htab_elem *) (htab->elems + i * htab->elem_size); -++} -++ -++static void htab_free_elems(struct bpf_htab *htab) -++{ -++ int i; -++ -++ if (!htab_is_percpu(htab)) -++ goto free_elems; -++ -++ for (i = 0; i < htab->map.max_entries; i++) { -++ void __percpu *pptr; -++ -++ pptr = htab_elem_get_ptr(get_htab_elem(htab, i), -++ htab->map.key_size); -++ free_percpu(pptr); -++ cond_resched(); -++ } -++free_elems: -++ bpf_map_area_free(htab->elems); -++} -++ -++static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key, -++ u32 hash) -++{ -++ struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash); -++ struct htab_elem *l; -++ -++ if (node) { -++ l = container_of(node, struct htab_elem, lru_node); -++ memcpy(l->key, key, htab->map.key_size); -++ return l; -++ } -++ -++ return NULL; -++} -++ -++static int prealloc_init(struct bpf_htab *htab) -++{ -++ u32 num_entries = htab->map.max_entries; -++ int err = -ENOMEM, i; -++ -++ if (!htab_is_percpu(htab) && !htab_is_lru(htab)) -++ num_entries += num_possible_cpus(); -++ -++ htab->elems = bpf_map_area_alloc(htab->elem_size * num_entries, -++ htab->map.numa_node); -++ if (!htab->elems) -++ return -ENOMEM; -++ -++ if (!htab_is_percpu(htab)) -++ goto skip_percpu_elems; -++ -++ for (i = 0; i < num_entries; i++) { -++ u32 size = round_up(htab->map.value_size, 8); -++ void __percpu *pptr; -++ -++ pptr = __alloc_percpu_gfp(size, 8, GFP_USER | __GFP_NOWARN); -++ if (!pptr) -++ goto free_elems; -++ htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, -++ pptr); -++ cond_resched(); -++ } -++ -++skip_percpu_elems: -++ if (htab_is_lru(htab)) -++ err = bpf_lru_init(&htab->lru, -++ htab->map.map_flags & BPF_F_NO_COMMON_LRU, -++ offsetof(struct htab_elem, hash) - -++ offsetof(struct htab_elem, lru_node), -++ htab_lru_map_delete_node, -++ htab); -++ else -++ err = pcpu_freelist_init(&htab->freelist); -++ -++ if (err) -++ goto free_elems; -++ -++ if (htab_is_lru(htab)) -++ bpf_lru_populate(&htab->lru, htab->elems, -++ offsetof(struct htab_elem, lru_node), -++ htab->elem_size, num_entries); -++ else -++ pcpu_freelist_populate(&htab->freelist, -++ htab->elems + offsetof(struct htab_elem, fnode), -++ htab->elem_size, num_entries); -++ -++ return 0; -++ -++free_elems: -++ htab_free_elems(htab); -++ return err; -++} -++ -++static void prealloc_destroy(struct bpf_htab *htab) -++{ -++ htab_free_elems(htab); -++ -++ if (htab_is_lru(htab)) -++ bpf_lru_destroy(&htab->lru); -++ else -++ pcpu_freelist_destroy(&htab->freelist); -++} -++ -++static int alloc_extra_elems(struct bpf_htab *htab) -++{ -++ struct htab_elem *__percpu *pptr, *l_new; -++ struct pcpu_freelist_node *l; -++ int cpu; -++ -++ pptr = __alloc_percpu_gfp(sizeof(struct htab_elem *), 8, -++ GFP_USER | __GFP_NOWARN); -++ if (!pptr) -++ return -ENOMEM; -++ -++ for_each_possible_cpu(cpu) { -++ l = pcpu_freelist_pop(&htab->freelist); -++ /* pop will succeed, since prealloc_init() -++ * preallocated extra num_possible_cpus elements -++ */ -++ l_new = container_of(l, struct htab_elem, fnode); -++ *per_cpu_ptr(pptr, cpu) = l_new; -++ } -++ htab->extra_elems = pptr; -++ return 0; -++} -++ -+ /* Called from syscall */ -+-static struct bpf_map *htab_map_alloc(union bpf_attr *attr) -++static int htab_map_alloc_check(union bpf_attr *attr) -+ { -+- struct bpf_htab *htab; -+- int err, i; -++ bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); -++ bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || -++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); -++ /* percpu_lru means each cpu has its own LRU list. -++ * it is different from BPF_MAP_TYPE_PERCPU_HASH where -++ * the map's value itself is percpu. percpu_lru has -++ * nothing to do with the map's value. -++ */ -++ bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); -++ bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); -++ bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED); -++ int numa_node = bpf_map_attr_numa_node(attr); -++ -++ BUILD_BUG_ON(offsetof(struct htab_elem, htab) != -++ offsetof(struct htab_elem, hash_node.pprev)); -++ BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) != -++ offsetof(struct htab_elem, hash_node.pprev)); -++ -++ if (lru && !capable(CAP_SYS_ADMIN)) -++ /* LRU implementation is much complicated than other -++ * maps. Hence, limit to CAP_SYS_ADMIN for now. -++ */ -++ return -EPERM; -+ -+- htab = kzalloc(sizeof(*htab), GFP_USER); -+- if (!htab) -+- return ERR_PTR(-ENOMEM); -++ if (zero_seed && !capable(CAP_SYS_ADMIN)) -++ /* Guard against local DoS, and discourage production use. */ -++ return -EPERM; -+ -+- /* mandatory map attributes */ -+- htab->map.key_size = attr->key_size; -+- htab->map.value_size = attr->value_size; -+- htab->map.max_entries = attr->max_entries; -++ if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK || -++ !bpf_map_flags_access_ok(attr->map_flags)) -++ return -EINVAL; -++ -++ if (!lru && percpu_lru) -++ return -EINVAL; -++ -++ if (lru && !prealloc) -++ return -ENOTSUPP; -++ -++ if (numa_node != NUMA_NO_NODE && (percpu || percpu_lru)) -++ return -EINVAL; -+ -+ /* check sanity of attributes. -+ * value_size == 0 may be allowed in the future to use map as a set -+ */ -+- err = -EINVAL; -+- if (htab->map.max_entries == 0 || htab->map.key_size == 0 || -+- htab->map.value_size == 0) -+- goto free_htab; -+- -+- /* hash table size must be power of 2 */ -+- htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); -++ if (attr->max_entries == 0 || attr->key_size == 0 || -++ attr->value_size == 0) -++ return -EINVAL; -+ -+- err = -E2BIG; -+- if (htab->map.key_size > MAX_BPF_STACK) -++ if (attr->key_size > MAX_BPF_STACK) -+ /* eBPF programs initialize keys on stack, so they cannot be -+ * larger than max stack size -+ */ -+- goto free_htab; -++ return -E2BIG; -+ -+- if (htab->map.value_size >= (1 << (KMALLOC_SHIFT_MAX - 1)) - -++ if (attr->value_size >= KMALLOC_MAX_SIZE - -+ MAX_BPF_STACK - sizeof(struct htab_elem)) -+ /* if value_size is bigger, the user space won't be able to -+ * access the elements via bpf syscall. This check also makes -+ * sure that the elem_size doesn't overflow and it's -+ * kmalloc-able later in htab_map_update_elem() -+ */ -+- goto free_htab; -++ return -E2BIG; -++ -++ return 0; -++} -++ -++static struct bpf_map *htab_map_alloc(union bpf_attr *attr) -++{ -++ bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); -++ bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH || -++ attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH); -++ /* percpu_lru means each cpu has its own LRU list. -++ * it is different from BPF_MAP_TYPE_PERCPU_HASH where -++ * the map's value itself is percpu. percpu_lru has -++ * nothing to do with the map's value. -++ */ -++ bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU); -++ bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC); -++ struct bpf_htab *htab; -++ int err, i; -++ u64 cost; -++ -++ htab = kzalloc(sizeof(*htab), GFP_USER); -++ if (!htab) -++ return ERR_PTR(-ENOMEM); -++ -++ bpf_map_init_from_attr(&htab->map, attr); -++ -++ if (percpu_lru) { -++ /* ensure each CPU's lru list has >=1 elements. -++ * since we are at it, make each lru list has the same -++ * number of elements. -++ */ -++ htab->map.max_entries = roundup(attr->max_entries, -++ num_possible_cpus()); -++ if (htab->map.max_entries < attr->max_entries) -++ htab->map.max_entries = rounddown(attr->max_entries, -++ num_possible_cpus()); -++ } -++ -++ /* hash table size must be power of 2 */ -++ htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); -+ -+ htab->elem_size = sizeof(struct htab_elem) + -+- round_up(htab->map.key_size, 8) + -+- htab->map.value_size; -++ round_up(htab->map.key_size, 8); -++ if (percpu) -++ htab->elem_size += sizeof(void *); -++ else -++ htab->elem_size += round_up(htab->map.value_size, 8); -+ -++ err = -E2BIG; -+ /* prevent zero size kmalloc and check for u32 overflow */ -+ if (htab->n_buckets == 0 || -+- htab->n_buckets > U32_MAX / sizeof(struct hlist_head)) -++ htab->n_buckets > U32_MAX / sizeof(struct bucket)) -+ goto free_htab; -+ -+- if ((u64) htab->n_buckets * sizeof(struct hlist_head) + -+- (u64) htab->elem_size * htab->map.max_entries >= -+- U32_MAX - PAGE_SIZE) -+- /* make sure page count doesn't overflow */ -+- goto free_htab; -++ cost = (u64) htab->n_buckets * sizeof(struct bucket) + -++ (u64) htab->elem_size * htab->map.max_entries; -+ -+- htab->map.pages = round_up(htab->n_buckets * sizeof(struct hlist_head) + -+- htab->elem_size * htab->map.max_entries, -+- PAGE_SIZE) >> PAGE_SHIFT; -++ if (percpu) -++ cost += (u64) round_up(htab->map.value_size, 8) * -++ num_possible_cpus() * htab->map.max_entries; -++ else -++ cost += (u64) htab->elem_size * num_possible_cpus(); -++ -++ /* if map size is larger than memlock limit, reject it */ -++ err = bpf_map_charge_init(&htab->map.memory, cost); -++ if (err) -++ goto free_htab; -+ -+ err = -ENOMEM; -+- htab->buckets = kmalloc_array(htab->n_buckets, sizeof(struct hlist_head), -+- GFP_USER | __GFP_NOWARN); -++ htab->buckets = bpf_map_area_alloc(htab->n_buckets * -++ sizeof(struct bucket), -++ htab->map.numa_node); -++ if (!htab->buckets) -++ goto free_charge; -++ -++ if (htab->map.map_flags & BPF_F_ZERO_SEED) -++ htab->hashrnd = 0; -++ else -++ htab->hashrnd = get_random_int(); -+ -+- if (!htab->buckets) { -+- htab->buckets = vmalloc(htab->n_buckets * sizeof(struct hlist_head)); -+- if (!htab->buckets) -+- goto free_htab; -++ for (i = 0; i < htab->n_buckets; i++) { -++ INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); -++ raw_spin_lock_init(&htab->buckets[i].lock); -+ } -+ -+- for (i = 0; i < htab->n_buckets; i++) -+- INIT_HLIST_HEAD(&htab->buckets[i]); -+- -+- raw_spin_lock_init(&htab->lock); -+- htab->count = 0; -++ if (prealloc) { -++ err = prealloc_init(htab); -++ if (err) -++ goto free_buckets; -++ -++ if (!percpu && !lru) { -++ /* lru itself can remove the least used element, so -++ * there is no need for an extra elem during map_update. -++ */ -++ err = alloc_extra_elems(htab); -++ if (err) -++ goto free_prealloc; -++ } -++ } -+ -+ return &htab->map; -+ -++free_prealloc: -++ prealloc_destroy(htab); -++free_buckets: -++ bpf_map_area_free(htab->buckets); -++free_charge: -++ bpf_map_charge_finish(&htab->map.memory); -+ free_htab: -+ kfree(htab); -+ return ERR_PTR(err); -+ } -+ -+-static inline u32 htab_map_hash(const void *key, u32 key_len) -++static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd) -+ { -+- return jhash(key, key_len, 0); -++ return jhash(key, key_len, hashrnd); -+ } -+ -+-static inline struct hlist_head *select_bucket(struct bpf_htab *htab, u32 hash) -++static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash) -+ { -+ return &htab->buckets[hash & (htab->n_buckets - 1)]; -+ } -+ -+-static struct htab_elem *lookup_elem_raw(struct hlist_head *head, u32 hash, -++static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash) -++{ -++ return &__select_bucket(htab, hash)->head; -++} -++ -++/* this lookup function can only be called with bucket lock taken */ -++static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash, -+ void *key, u32 key_size) -+ { -++ struct hlist_nulls_node *n; -+ struct htab_elem *l; -+ -+- hlist_for_each_entry_rcu(l, head, hash_node) -++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) -+ if (l->hash == hash && !memcmp(&l->key, key, key_size)) -+ return l; -+ -+ return NULL; -+ } -+ -+-/* Called from syscall or from eBPF program */ -+-static void *htab_map_lookup_elem(struct bpf_map *map, void *key) -++/* can be called without bucket lock. it will repeat the loop in -++ * the unlikely event when elements moved from one bucket into another -++ * while link list is being walked -++ */ -++static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head, -++ u32 hash, void *key, -++ u32 key_size, u32 n_buckets) -++{ -++ struct hlist_nulls_node *n; -++ struct htab_elem *l; -++ -++again: -++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) -++ if (l->hash == hash && !memcmp(&l->key, key, key_size)) -++ return l; -++ -++ if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1)))) -++ goto again; -++ -++ return NULL; -++} -++ -++/* Called from syscall or from eBPF program directly, so -++ * arguments have to match bpf_map_lookup_elem() exactly. -++ * The return value is adjusted by BPF instructions -++ * in htab_map_gen_lookup(). -++ */ -++static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) -+ { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+- struct hlist_head *head; -++ struct hlist_nulls_head *head; -+ struct htab_elem *l; -+ u32 hash, key_size; -+ -+@@ -150,11 +470,18 @@ static void *htab_map_lookup_elem(struct -+ -+ key_size = map->key_size; -+ -+- hash = htab_map_hash(key, key_size); -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -+ -+ head = select_bucket(htab, hash); -+ -+- l = lookup_elem_raw(head, hash, key, key_size); -++ l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); -++ -++ return l; -++} -++ -++static void *htab_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct htab_elem *l = __htab_map_lookup_elem(map, key); -+ -+ if (l) -+ return l->key + round_up(map->key_size, 8); -+@@ -162,33 +489,138 @@ static void *htab_map_lookup_elem(struct -+ return NULL; -+ } -+ -++/* inline bpf_map_lookup_elem() call. -++ * Instead of: -++ * bpf_prog -++ * bpf_map_lookup_elem -++ * map->ops->map_lookup_elem -++ * htab_map_lookup_elem -++ * __htab_map_lookup_elem -++ * do: -++ * bpf_prog -++ * __htab_map_lookup_elem -++ */ -++static u32 htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) -++{ -++ struct bpf_insn *insn = insn_buf; -++ const int ret = BPF_REG_0; -++ -++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, -++ (void *(*)(struct bpf_map *map, void *key))NULL)); -++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1); -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, -++ offsetof(struct htab_elem, key) + -++ round_up(map->key_size, 8)); -++ return insn - insn_buf; -++} -++ -++static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, -++ void *key, const bool mark) -++{ -++ struct htab_elem *l = __htab_map_lookup_elem(map, key); -++ -++ if (l) { -++ if (mark) -++ bpf_lru_node_set_ref(&l->lru_node); -++ return l->key + round_up(map->key_size, 8); -++ } -++ -++ return NULL; -++} -++ -++static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ return __htab_lru_map_lookup_elem(map, key, true); -++} -++ -++static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) -++{ -++ return __htab_lru_map_lookup_elem(map, key, false); -++} -++ -++static u32 htab_lru_map_gen_lookup(struct bpf_map *map, -++ struct bpf_insn *insn_buf) -++{ -++ struct bpf_insn *insn = insn_buf; -++ const int ret = BPF_REG_0; -++ const int ref_reg = BPF_REG_1; -++ -++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, -++ (void *(*)(struct bpf_map *map, void *key))NULL)); -++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4); -++ *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret, -++ offsetof(struct htab_elem, lru_node) + -++ offsetof(struct bpf_lru_node, ref)); -++ *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1); -++ *insn++ = BPF_ST_MEM(BPF_B, ret, -++ offsetof(struct htab_elem, lru_node) + -++ offsetof(struct bpf_lru_node, ref), -++ 1); -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, -++ offsetof(struct htab_elem, key) + -++ round_up(map->key_size, 8)); -++ return insn - insn_buf; -++} -++ -++/* It is called from the bpf_lru_list when the LRU needs to delete -++ * older elements from the htab. -++ */ -++static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node) -++{ -++ struct bpf_htab *htab = (struct bpf_htab *)arg; -++ struct htab_elem *l = NULL, *tgt_l; -++ struct hlist_nulls_head *head; -++ struct hlist_nulls_node *n; -++ unsigned long flags; -++ struct bucket *b; -++ -++ tgt_l = container_of(node, struct htab_elem, lru_node); -++ b = __select_bucket(htab, tgt_l->hash); -++ head = &b->head; -++ -++ raw_spin_lock_irqsave(&b->lock, flags); -++ -++ hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) -++ if (l == tgt_l) { -++ hlist_nulls_del_rcu(&l->hash_node); -++ break; -++ } -++ -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ -++ return l == tgt_l; -++} -++ -+ /* Called from syscall */ -+ static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) -+ { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+- struct hlist_head *head; -++ struct hlist_nulls_head *head; -+ struct htab_elem *l, *next_l; -+ u32 hash, key_size; -+- int i; -++ int i = 0; -+ -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+ -+ key_size = map->key_size; -+ -+- hash = htab_map_hash(key, key_size); -++ if (!key) -++ goto find_first_elem; -++ -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -+ -+ head = select_bucket(htab, hash); -+ -+ /* lookup the key */ -+- l = lookup_elem_raw(head, hash, key, key_size); -++ l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); -+ -+- if (!l) { -+- i = 0; -++ if (!l) -+ goto find_first_elem; -+- } -+ -+ /* key was found, get next key in the same bucket */ -+- next_l = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu(&l->hash_node)), -++ next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)), -+ struct htab_elem, hash_node); -+ -+ if (next_l) { -+@@ -207,7 +639,7 @@ find_first_elem: -+ head = select_bucket(htab, i); -+ -+ /* pick first element in the bucket */ -+- next_l = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(head)), -++ next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)), -+ struct htab_elem, hash_node); -+ if (next_l) { -+ /* if it's not empty, just return it */ -+@@ -216,90 +648,491 @@ find_first_elem: -+ } -+ } -+ -+- /* itereated over all buckets and all elements */ -++ /* iterated over all buckets and all elements */ -+ return -ENOENT; -+ } -+ -++static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) -++{ -++ if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) -++ free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); -++ kfree(l); -++} -++ -++static void htab_elem_free_rcu(struct rcu_head *head) -++{ -++ struct htab_elem *l = container_of(head, struct htab_elem, rcu); -++ struct bpf_htab *htab = l->htab; -++ -++ htab_elem_free(htab, l); -++} -++ -++static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) -++{ -++ struct bpf_map *map = &htab->map; -++ void *ptr; -++ -++ if (map->ops->map_fd_put_ptr) { -++ ptr = fd_htab_map_get_ptr(map, l); -++ map->ops->map_fd_put_ptr(ptr); -++ } -++} -++ -++static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) -++{ -++ htab_put_fd_value(htab, l); -++ -++ if (htab_is_prealloc(htab)) { -++ __pcpu_freelist_push(&htab->freelist, &l->fnode); -++ } else { -++ atomic_dec(&htab->count); -++ l->htab = htab; -++ call_rcu(&l->rcu, htab_elem_free_rcu); -++ } -++} -++ -++static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, -++ void *value, bool onallcpus) -++{ -++ if (!onallcpus) { -++ /* copy true value_size bytes */ -++ memcpy(this_cpu_ptr(pptr), value, htab->map.value_size); -++ } else { -++ u32 size = round_up(htab->map.value_size, 8); -++ int off = 0, cpu; -++ -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), -++ value + off, size); -++ off += size; -++ } -++ } -++} -++ -++static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, -++ void *value, bool onallcpus) -++{ -++ /* When using prealloc and not setting the initial value on all cpus, -++ * zero-fill element values for other cpus (just as what happens when -++ * not using prealloc). Otherwise, bpf program has no way to ensure -++ * known initial values for cpus other than current one -++ * (onallcpus=false always when coming from bpf prog). -++ */ -++ if (htab_is_prealloc(htab) && !onallcpus) { -++ u32 size = round_up(htab->map.value_size, 8); -++ int current_cpu = raw_smp_processor_id(); -++ int cpu; -++ -++ for_each_possible_cpu(cpu) { -++ if (cpu == current_cpu) -++ bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value, -++ size); -++ else -++ memset(per_cpu_ptr(pptr, cpu), 0, size); -++ } -++ } else { -++ pcpu_copy_value(htab, pptr, value, onallcpus); -++ } -++} -++ -++static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab) -++{ -++ return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS && -++ BITS_PER_LONG == 64; -++} -++ -++static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, -++ void *value, u32 key_size, u32 hash, -++ bool percpu, bool onallcpus, -++ struct htab_elem *old_elem) -++{ -++ u32 size = htab->map.value_size; -++ bool prealloc = htab_is_prealloc(htab); -++ struct htab_elem *l_new, **pl_new; -++ void __percpu *pptr; -++ -++ if (prealloc) { -++ if (old_elem) { -++ /* if we're updating the existing element, -++ * use per-cpu extra elems to avoid freelist_pop/push -++ */ -++ pl_new = this_cpu_ptr(htab->extra_elems); -++ l_new = *pl_new; -++ htab_put_fd_value(htab, old_elem); -++ *pl_new = old_elem; -++ } else { -++ struct pcpu_freelist_node *l; -++ -++ l = __pcpu_freelist_pop(&htab->freelist); -++ if (!l) -++ return ERR_PTR(-E2BIG); -++ l_new = container_of(l, struct htab_elem, fnode); -++ } -++ } else { -++ if (atomic_inc_return(&htab->count) > htab->map.max_entries) -++ if (!old_elem) { -++ /* when map is full and update() is replacing -++ * old element, it's ok to allocate, since -++ * old element will be freed immediately. -++ * Otherwise return an error -++ */ -++ l_new = ERR_PTR(-E2BIG); -++ goto dec_count; -++ } -++ l_new = kmalloc_node(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN, -++ htab->map.numa_node); -++ if (!l_new) { -++ l_new = ERR_PTR(-ENOMEM); -++ goto dec_count; -++ } -++ check_and_init_map_lock(&htab->map, -++ l_new->key + round_up(key_size, 8)); -++ } -++ -++ memcpy(l_new->key, key, key_size); -++ if (percpu) { -++ size = round_up(size, 8); -++ if (prealloc) { -++ pptr = htab_elem_get_ptr(l_new, key_size); -++ } else { -++ /* alloc_percpu zero-fills */ -++ pptr = __alloc_percpu_gfp(size, 8, -++ GFP_ATOMIC | __GFP_NOWARN); -++ if (!pptr) { -++ kfree(l_new); -++ l_new = ERR_PTR(-ENOMEM); -++ goto dec_count; -++ } -++ } -++ -++ pcpu_init_value(htab, pptr, value, onallcpus); -++ -++ if (!prealloc) -++ htab_elem_set_ptr(l_new, key_size, pptr); -++ } else if (fd_htab_map_needs_adjust(htab)) { -++ size = round_up(size, 8); -++ memcpy(l_new->key + round_up(key_size, 8), value, size); -++ } else { -++ copy_map_value(&htab->map, -++ l_new->key + round_up(key_size, 8), -++ value); -++ } -++ -++ l_new->hash = hash; -++ return l_new; -++dec_count: -++ atomic_dec(&htab->count); -++ return l_new; -++} -++ -++static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old, -++ u64 map_flags) -++{ -++ if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST) -++ /* elem already exists */ -++ return -EEXIST; -++ -++ if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST) -++ /* elem doesn't exist, cannot update it */ -++ return -ENOENT; -++ -++ return 0; -++} -++ -+ /* Called from syscall or from eBPF program */ -+ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value, -+ u64 map_flags) -+ { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+- struct htab_elem *l_new, *l_old; -+- struct hlist_head *head; -++ struct htab_elem *l_new = NULL, *l_old; -++ struct hlist_nulls_head *head; -+ unsigned long flags; -+- u32 key_size; -++ struct bucket *b; -++ u32 key_size, hash; -+ int ret; -+ -+- if (map_flags > BPF_EXIST) -++ if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST)) -+ /* unknown flags */ -+ return -EINVAL; -+ -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+ -+- /* allocate new element outside of lock */ -+- l_new = kmalloc(htab->elem_size, GFP_ATOMIC | __GFP_NOWARN); -+- if (!l_new) -+- return -ENOMEM; -+- -+ key_size = map->key_size; -+ -+- memcpy(l_new->key, key, key_size); -+- memcpy(l_new->key + round_up(key_size, 8), value, map->value_size); -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ -++ b = __select_bucket(htab, hash); -++ head = &b->head; -+ -+- l_new->hash = htab_map_hash(l_new->key, key_size); -++ if (unlikely(map_flags & BPF_F_LOCK)) { -++ if (unlikely(!map_value_has_spin_lock(map))) -++ return -EINVAL; -++ /* find an element without taking the bucket lock */ -++ l_old = lookup_nulls_elem_raw(head, hash, key, key_size, -++ htab->n_buckets); -++ ret = check_flags(htab, l_old, map_flags); -++ if (ret) -++ return ret; -++ if (l_old) { -++ /* grab the element lock and update value in place */ -++ copy_map_value_locked(map, -++ l_old->key + round_up(key_size, 8), -++ value, false); -++ return 0; -++ } -++ /* fall through, grab the bucket lock and lookup again. -++ * 99.9% chance that the element won't be found, -++ * but second lookup under lock has to be done. -++ */ -++ } -+ -+ /* bpf_map_update_elem() can be called in_irq() */ -+- raw_spin_lock_irqsave(&htab->lock, flags); -++ raw_spin_lock_irqsave(&b->lock, flags); -+ -+- head = select_bucket(htab, l_new->hash); -++ l_old = lookup_elem_raw(head, hash, key, key_size); -+ -+- l_old = lookup_elem_raw(head, l_new->hash, key, key_size); -++ ret = check_flags(htab, l_old, map_flags); -++ if (ret) -++ goto err; -+ -+- if (!l_old && unlikely(htab->count >= map->max_entries)) { -+- /* if elem with this 'key' doesn't exist and we've reached -+- * max_entries limit, fail insertion of new elem -++ if (unlikely(l_old && (map_flags & BPF_F_LOCK))) { -++ /* first lookup without the bucket lock didn't find the element, -++ * but second lookup with the bucket lock found it. -++ * This case is highly unlikely, but has to be dealt with: -++ * grab the element lock in addition to the bucket lock -++ * and update element in place -+ */ -+- ret = -E2BIG; -++ copy_map_value_locked(map, -++ l_old->key + round_up(key_size, 8), -++ value, false); -++ ret = 0; -+ goto err; -+ } -+ -+- if (l_old && map_flags == BPF_NOEXIST) { -+- /* elem already exists */ -+- ret = -EEXIST; -++ l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false, -++ l_old); -++ if (IS_ERR(l_new)) { -++ /* all pre-allocated elements are in use or memory exhausted */ -++ ret = PTR_ERR(l_new); -+ goto err; -+ } -+ -+- if (!l_old && map_flags == BPF_EXIST) { -+- /* elem doesn't exist, cannot update it */ -+- ret = -ENOENT; -+- goto err; -++ /* add new element to the head of the list, so that -++ * concurrent search will find it before old elem -++ */ -++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); -++ if (l_old) { -++ hlist_nulls_del_rcu(&l_old->hash_node); -++ if (!htab_is_prealloc(htab)) -++ free_htab_elem(htab, l_old); -+ } -++ ret = 0; -++err: -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ return ret; -++} -+ -+- /* add new element to the head of the list, so that concurrent -+- * search will find it before old elem -++static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ struct htab_elem *l_new, *l_old = NULL; -++ struct hlist_nulls_head *head; -++ unsigned long flags; -++ struct bucket *b; -++ u32 key_size, hash; -++ int ret; -++ -++ if (unlikely(map_flags > BPF_EXIST)) -++ /* unknown flags */ -++ return -EINVAL; -++ -++ WARN_ON_ONCE(!rcu_read_lock_held()); -++ -++ key_size = map->key_size; -++ -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ -++ b = __select_bucket(htab, hash); -++ head = &b->head; -++ -++ /* For LRU, we need to alloc before taking bucket's -++ * spinlock because getting free nodes from LRU may need -++ * to remove older elements from htab and this removal -++ * operation will need a bucket lock. -+ */ -+- hlist_add_head_rcu(&l_new->hash_node, head); -++ l_new = prealloc_lru_pop(htab, key, hash); -++ if (!l_new) -++ return -ENOMEM; -++ memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size); -++ -++ /* bpf_map_update_elem() can be called in_irq() */ -++ raw_spin_lock_irqsave(&b->lock, flags); -++ -++ l_old = lookup_elem_raw(head, hash, key, key_size); -++ -++ ret = check_flags(htab, l_old, map_flags); -++ if (ret) -++ goto err; -++ -++ /* add new element to the head of the list, so that -++ * concurrent search will find it before old elem -++ */ -++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); -++ if (l_old) { -++ bpf_lru_node_set_ref(&l_new->lru_node); -++ hlist_nulls_del_rcu(&l_old->hash_node); -++ } -++ ret = 0; -++ -++err: -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ -++ if (ret) -++ bpf_lru_push_free(&htab->lru, &l_new->lru_node); -++ else if (l_old) -++ bpf_lru_push_free(&htab->lru, &l_old->lru_node); -++ -++ return ret; -++} -++ -++static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 map_flags, -++ bool onallcpus) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ struct htab_elem *l_new = NULL, *l_old; -++ struct hlist_nulls_head *head; -++ unsigned long flags; -++ struct bucket *b; -++ u32 key_size, hash; -++ int ret; -++ -++ if (unlikely(map_flags > BPF_EXIST)) -++ /* unknown flags */ -++ return -EINVAL; -++ -++ WARN_ON_ONCE(!rcu_read_lock_held()); -++ -++ key_size = map->key_size; -++ -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ -++ b = __select_bucket(htab, hash); -++ head = &b->head; -++ -++ /* bpf_map_update_elem() can be called in_irq() */ -++ raw_spin_lock_irqsave(&b->lock, flags); -++ -++ l_old = lookup_elem_raw(head, hash, key, key_size); -++ -++ ret = check_flags(htab, l_old, map_flags); -++ if (ret) -++ goto err; -++ -+ if (l_old) { -+- hlist_del_rcu(&l_old->hash_node); -+- kfree_rcu(l_old, rcu); -++ /* per-cpu hash map can update value in-place */ -++ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), -++ value, onallcpus); -+ } else { -+- htab->count++; -++ l_new = alloc_htab_elem(htab, key, value, key_size, -++ hash, true, onallcpus, NULL); -++ if (IS_ERR(l_new)) { -++ ret = PTR_ERR(l_new); -++ goto err; -++ } -++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); -+ } -+- raw_spin_unlock_irqrestore(&htab->lock, flags); -++ ret = 0; -++err: -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ return ret; -++} -+ -+- return 0; -++static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 map_flags, -++ bool onallcpus) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ struct htab_elem *l_new = NULL, *l_old; -++ struct hlist_nulls_head *head; -++ unsigned long flags; -++ struct bucket *b; -++ u32 key_size, hash; -++ int ret; -++ -++ if (unlikely(map_flags > BPF_EXIST)) -++ /* unknown flags */ -++ return -EINVAL; -++ -++ WARN_ON_ONCE(!rcu_read_lock_held()); -++ -++ key_size = map->key_size; -++ -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ -++ b = __select_bucket(htab, hash); -++ head = &b->head; -++ -++ /* For LRU, we need to alloc before taking bucket's -++ * spinlock because LRU's elem alloc may need -++ * to remove older elem from htab and this removal -++ * operation will need a bucket lock. -++ */ -++ if (map_flags != BPF_EXIST) { -++ l_new = prealloc_lru_pop(htab, key, hash); -++ if (!l_new) -++ return -ENOMEM; -++ } -++ -++ /* bpf_map_update_elem() can be called in_irq() */ -++ raw_spin_lock_irqsave(&b->lock, flags); -++ -++ l_old = lookup_elem_raw(head, hash, key, key_size); -++ -++ ret = check_flags(htab, l_old, map_flags); -++ if (ret) -++ goto err; -++ -++ if (l_old) { -++ bpf_lru_node_set_ref(&l_old->lru_node); -++ -++ /* per-cpu hash map can update value in-place */ -++ pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size), -++ value, onallcpus); -++ } else { -++ pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size), -++ value, onallcpus); -++ hlist_nulls_add_head_rcu(&l_new->hash_node, head); -++ l_new = NULL; -++ } -++ ret = 0; -+ err: -+- raw_spin_unlock_irqrestore(&htab->lock, flags); -+- kfree(l_new); -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ if (l_new) -++ bpf_lru_push_free(&htab->lru, &l_new->lru_node); -+ return ret; -+ } -+ -++static int htab_percpu_map_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 map_flags) -++{ -++ return __htab_percpu_map_update_elem(map, key, value, map_flags, false); -++} -++ -++static int htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 map_flags) -++{ -++ return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, -++ false); -++} -++ -+ /* Called from syscall or from eBPF program */ -+ static int htab_map_delete_elem(struct bpf_map *map, void *key) -+ { -+ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -+- struct hlist_head *head; -++ struct hlist_nulls_head *head; -++ struct bucket *b; -+ struct htab_elem *l; -+ unsigned long flags; -+ u32 hash, key_size; -+@@ -309,22 +1142,54 @@ static int htab_map_delete_elem(struct b -+ -+ key_size = map->key_size; -+ -+- hash = htab_map_hash(key, key_size); -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ b = __select_bucket(htab, hash); -++ head = &b->head; -+ -+- raw_spin_lock_irqsave(&htab->lock, flags); -++ raw_spin_lock_irqsave(&b->lock, flags); -+ -+- head = select_bucket(htab, hash); -++ l = lookup_elem_raw(head, hash, key, key_size); -++ -++ if (l) { -++ hlist_nulls_del_rcu(&l->hash_node); -++ free_htab_elem(htab, l); -++ ret = 0; -++ } -++ -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ return ret; -++} -++ -++static int htab_lru_map_delete_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ struct hlist_nulls_head *head; -++ struct bucket *b; -++ struct htab_elem *l; -++ unsigned long flags; -++ u32 hash, key_size; -++ int ret = -ENOENT; -++ -++ WARN_ON_ONCE(!rcu_read_lock_held()); -++ -++ key_size = map->key_size; -++ -++ hash = htab_map_hash(key, key_size, htab->hashrnd); -++ b = __select_bucket(htab, hash); -++ head = &b->head; -++ -++ raw_spin_lock_irqsave(&b->lock, flags); -+ -+ l = lookup_elem_raw(head, hash, key, key_size); -+ -+ if (l) { -+- hlist_del_rcu(&l->hash_node); -+- htab->count--; -+- kfree_rcu(l, rcu); -++ hlist_nulls_del_rcu(&l->hash_node); -+ ret = 0; -+ } -+ -+- raw_spin_unlock_irqrestore(&htab->lock, flags); -++ raw_spin_unlock_irqrestore(&b->lock, flags); -++ if (l) -++ bpf_lru_push_free(&htab->lru, &l->lru_node); -+ return ret; -+ } -+ -+@@ -333,14 +1198,13 @@ static void delete_all_elements(struct b -+ int i; -+ -+ for (i = 0; i < htab->n_buckets; i++) { -+- struct hlist_head *head = select_bucket(htab, i); -+- struct hlist_node *n; -++ struct hlist_nulls_head *head = select_bucket(htab, i); -++ struct hlist_nulls_node *n; -+ struct htab_elem *l; -+ -+- hlist_for_each_entry_safe(l, n, head, hash_node) { -+- hlist_del_rcu(&l->hash_node); -+- htab->count--; -+- kfree(l); -++ hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { -++ hlist_nulls_del_rcu(&l->hash_node); -++ htab_elem_free(htab, l); -+ } -+ } -+ } -+@@ -357,31 +1221,320 @@ static void htab_map_free(struct bpf_map -+ */ -+ synchronize_rcu(); -+ -+- /* some of kfree_rcu() callbacks for elements of this map may not have -+- * executed. It's ok. Proceed to free residual elements and map itself -++ /* some of free_htab_elem() callbacks for elements of this map may -++ * not have executed. Wait for them. -+ */ -+- delete_all_elements(htab); -+- kvfree(htab->buckets); -++ rcu_barrier(); -++ if (!htab_is_prealloc(htab)) -++ delete_all_elements(htab); -++ else -++ prealloc_destroy(htab); -++ -++ free_percpu(htab->extra_elems); -++ bpf_map_area_free(htab->buckets); -+ kfree(htab); -+ } -+ -+-static const struct bpf_map_ops htab_ops = { -++static void htab_map_seq_show_elem(struct bpf_map *map, void *key, -++ struct seq_file *m) -++{ -++ void *value; -++ -++ rcu_read_lock(); -++ -++ value = htab_map_lookup_elem(map, key); -++ if (!value) { -++ rcu_read_unlock(); -++ return; -++ } -++ -++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); -++ seq_puts(m, ": "); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); -++ seq_puts(m, "\n"); -++ -++ rcu_read_unlock(); -++} -++ -++const struct bpf_map_ops htab_map_ops = { -++ .map_alloc_check = htab_map_alloc_check, -+ .map_alloc = htab_map_alloc, -+ .map_free = htab_map_free, -+ .map_get_next_key = htab_map_get_next_key, -+ .map_lookup_elem = htab_map_lookup_elem, -+ .map_update_elem = htab_map_update_elem, -+ .map_delete_elem = htab_map_delete_elem, -++ .map_gen_lookup = htab_map_gen_lookup, -++ .map_seq_show_elem = htab_map_seq_show_elem, -+ }; -+ -+-static struct bpf_map_type_list htab_type __read_mostly = { -+- .ops = &htab_ops, -+- .type = BPF_MAP_TYPE_HASH, -++const struct bpf_map_ops htab_lru_map_ops = { -++ .map_alloc_check = htab_map_alloc_check, -++ .map_alloc = htab_map_alloc, -++ .map_free = htab_map_free, -++ .map_get_next_key = htab_map_get_next_key, -++ .map_lookup_elem = htab_lru_map_lookup_elem, -++ .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys, -++ .map_update_elem = htab_lru_map_update_elem, -++ .map_delete_elem = htab_lru_map_delete_elem, -++ .map_gen_lookup = htab_lru_map_gen_lookup, -++ .map_seq_show_elem = htab_map_seq_show_elem, -+ }; -+ -+-static int __init register_htab_map(void) -++/* Called from eBPF program */ -++static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) -+ { -+- bpf_register_map_type(&htab_type); -+- return 0; -++ struct htab_elem *l = __htab_map_lookup_elem(map, key); -++ -++ if (l) -++ return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -++ else -++ return NULL; -+ } -+-late_initcall(register_htab_map); -++ -++static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct htab_elem *l = __htab_map_lookup_elem(map, key); -++ -++ if (l) { -++ bpf_lru_node_set_ref(&l->lru_node); -++ return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); -++ } -++ -++ return NULL; -++} -++ -++int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) -++{ -++ struct htab_elem *l; -++ void __percpu *pptr; -++ int ret = -ENOENT; -++ int cpu, off = 0; -++ u32 size; -++ -++ /* per_cpu areas are zero-filled and bpf programs can only -++ * access 'value_size' of them, so copying rounded areas -++ * will not leak any kernel data -++ */ -++ size = round_up(map->value_size, 8); -++ rcu_read_lock(); -++ l = __htab_map_lookup_elem(map, key); -++ if (!l) -++ goto out; -++ /* We do not mark LRU map element here in order to not mess up -++ * eviction heuristics when user space does a map walk. -++ */ -++ pptr = htab_elem_get_ptr(l, map->key_size); -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(value + off, -++ per_cpu_ptr(pptr, cpu), size); -++ off += size; -++ } -++ ret = 0; -++out: -++ rcu_read_unlock(); -++ return ret; -++} -++ -++int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ int ret; -++ -++ rcu_read_lock(); -++ if (htab_is_lru(htab)) -++ ret = __htab_lru_percpu_map_update_elem(map, key, value, -++ map_flags, true); -++ else -++ ret = __htab_percpu_map_update_elem(map, key, value, map_flags, -++ true); -++ rcu_read_unlock(); -++ -++ return ret; -++} -++ -++static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, -++ struct seq_file *m) -++{ -++ struct htab_elem *l; -++ void __percpu *pptr; -++ int cpu; -++ -++ rcu_read_lock(); -++ -++ l = __htab_map_lookup_elem(map, key); -++ if (!l) { -++ rcu_read_unlock(); -++ return; -++ } -++ -++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); -++ seq_puts(m, ": {\n"); -++ pptr = htab_elem_get_ptr(l, map->key_size); -++ for_each_possible_cpu(cpu) { -++ seq_printf(m, "\tcpu%d: ", cpu); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, -++ per_cpu_ptr(pptr, cpu), m); -++ seq_puts(m, "\n"); -++ } -++ seq_puts(m, "}\n"); -++ -++ rcu_read_unlock(); -++} -++ -++const struct bpf_map_ops htab_percpu_map_ops = { -++ .map_alloc_check = htab_map_alloc_check, -++ .map_alloc = htab_map_alloc, -++ .map_free = htab_map_free, -++ .map_get_next_key = htab_map_get_next_key, -++ .map_lookup_elem = htab_percpu_map_lookup_elem, -++ .map_update_elem = htab_percpu_map_update_elem, -++ .map_delete_elem = htab_map_delete_elem, -++ .map_seq_show_elem = htab_percpu_map_seq_show_elem, -++}; -++ -++const struct bpf_map_ops htab_lru_percpu_map_ops = { -++ .map_alloc_check = htab_map_alloc_check, -++ .map_alloc = htab_map_alloc, -++ .map_free = htab_map_free, -++ .map_get_next_key = htab_map_get_next_key, -++ .map_lookup_elem = htab_lru_percpu_map_lookup_elem, -++ .map_update_elem = htab_lru_percpu_map_update_elem, -++ .map_delete_elem = htab_lru_map_delete_elem, -++ .map_seq_show_elem = htab_percpu_map_seq_show_elem, -++}; -++ -++static int fd_htab_map_alloc_check(union bpf_attr *attr) -++{ -++ if (attr->value_size != sizeof(u32)) -++ return -EINVAL; -++ return htab_map_alloc_check(attr); -++} -++ -++static void fd_htab_map_free(struct bpf_map *map) -++{ -++ struct bpf_htab *htab = container_of(map, struct bpf_htab, map); -++ struct hlist_nulls_node *n; -++ struct hlist_nulls_head *head; -++ struct htab_elem *l; -++ int i; -++ -++ for (i = 0; i < htab->n_buckets; i++) { -++ head = select_bucket(htab, i); -++ -++ hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { -++ void *ptr = fd_htab_map_get_ptr(map, l); -++ -++ map->ops->map_fd_put_ptr(ptr); -++ } -++ } -++ -++ htab_map_free(map); -++} -++ -++/* only called from syscall */ -++int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) -++{ -++ void **ptr; -++ int ret = 0; -++ -++ if (!map->ops->map_fd_sys_lookup_elem) -++ return -ENOTSUPP; -++ -++ rcu_read_lock(); -++ ptr = htab_map_lookup_elem(map, key); -++ if (ptr) -++ *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); -++ else -++ ret = -ENOENT; -++ rcu_read_unlock(); -++ -++ return ret; -++} -++ -++/* only called from syscall */ -++int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, -++ void *key, void *value, u64 map_flags) -++{ -++ void *ptr; -++ int ret; -++ u32 ufd = *(u32 *)value; -++ -++ ptr = map->ops->map_fd_get_ptr(map, map_file, ufd); -++ if (IS_ERR(ptr)) -++ return PTR_ERR(ptr); -++ -++ ret = htab_map_update_elem(map, key, &ptr, map_flags); -++ if (ret) -++ map->ops->map_fd_put_ptr(ptr); -++ -++ return ret; -++} -++ -++static struct bpf_map *htab_of_map_alloc(union bpf_attr *attr) -++{ -++ struct bpf_map *map, *inner_map_meta; -++ -++ inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd); -++ if (IS_ERR(inner_map_meta)) -++ return inner_map_meta; -++ -++ map = htab_map_alloc(attr); -++ if (IS_ERR(map)) { -++ bpf_map_meta_free(inner_map_meta); -++ return map; -++ } -++ -++ map->inner_map_meta = inner_map_meta; -++ -++ return map; -++} -++ -++static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_map **inner_map = htab_map_lookup_elem(map, key); -++ -++ if (!inner_map) -++ return NULL; -++ -++ return READ_ONCE(*inner_map); -++} -++ -++static u32 htab_of_map_gen_lookup(struct bpf_map *map, -++ struct bpf_insn *insn_buf) -++{ -++ struct bpf_insn *insn = insn_buf; -++ const int ret = BPF_REG_0; -++ -++ BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem, -++ (void *(*)(struct bpf_map *map, void *key))NULL)); -++ *insn++ = BPF_EMIT_CALL(BPF_CAST_CALL(__htab_map_lookup_elem)); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2); -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, ret, -++ offsetof(struct htab_elem, key) + -++ round_up(map->key_size, 8)); -++ *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0); -++ -++ return insn - insn_buf; -++} -++ -++static void htab_of_map_free(struct bpf_map *map) -++{ -++ bpf_map_meta_free(map->inner_map_meta); -++ fd_htab_map_free(map); -++} -++ -++const struct bpf_map_ops htab_of_maps_map_ops = { -++ .map_alloc_check = fd_htab_map_alloc_check, -++ .map_alloc = htab_of_map_alloc, -++ .map_free = htab_of_map_free, -++ .map_get_next_key = htab_map_get_next_key, -++ .map_lookup_elem = htab_of_map_lookup_elem, -++ .map_delete_elem = htab_map_delete_elem, -++ .map_fd_get_ptr = bpf_map_fd_get_ptr, -++ .map_fd_put_ptr = bpf_map_fd_put_ptr, -++ .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem, -++ .map_gen_lookup = htab_of_map_gen_lookup, -++ .map_check_btf = map_check_no_btf, -++}; -+--- a/kernel/bpf/helpers.c -++++ b/kernel/bpf/helpers.c -+@@ -1,21 +1,18 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+- * -+- * This program is distributed in the hope that it will be useful, but -+- * WITHOUT ANY WARRANTY; without even the implied warranty of -+- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+- * General Public License for more details. -+ */ -+ #include -+ #include -+ #include -+ #include -++#include -+ #include -+ #include -+ #include -++#include -++#include -++ -++#include "../../lib/kstrtox.h" -+ -+ /* If kernel subsystem is allowing eBPF programs to call this function, -+ * inside its own verifier_ops->get_func_proto() callback it should return -+@@ -26,48 +23,32 @@ -+ * if program is allowed to access maps, so check rcu_read_lock_held in -+ * all three functions. -+ */ -+-static u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key) -+ { -+- /* verifier checked that R1 contains a valid pointer to bpf_map -+- * and R2 points to a program stack and map->key_size bytes were -+- * initialized -+- */ -+- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; -+- void *key = (void *) (unsigned long) r2; -+- void *value; -+- -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+- -+- value = map->ops->map_lookup_elem(map, key); -+- -+- /* lookup() returns either pointer to element value or NULL -+- * which is the meaning of PTR_TO_MAP_VALUE_OR_NULL type -+- */ -+- return (unsigned long) value; -++ return (unsigned long) map->ops->map_lookup_elem(map, key); -+ } -+ -+ const struct bpf_func_proto bpf_map_lookup_elem_proto = { -+ .func = bpf_map_lookup_elem, -+ .gpl_only = false, -++ .pkt_access = true, -+ .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, -+ .arg1_type = ARG_CONST_MAP_PTR, -+ .arg2_type = ARG_PTR_TO_MAP_KEY, -+ }; -+ -+-static u64 bpf_map_update_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key, -++ void *, value, u64, flags) -+ { -+- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; -+- void *key = (void *) (unsigned long) r2; -+- void *value = (void *) (unsigned long) r3; -+- -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+- -+- return map->ops->map_update_elem(map, key, value, r4); -++ return map->ops->map_update_elem(map, key, value, flags); -+ } -+ -+ const struct bpf_func_proto bpf_map_update_elem_proto = { -+ .func = bpf_map_update_elem, -+ .gpl_only = false, -++ .pkt_access = true, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_CONST_MAP_PTR, -+ .arg2_type = ARG_PTR_TO_MAP_KEY, -+@@ -75,33 +56,71 @@ const struct bpf_func_proto bpf_map_upda -+ .arg4_type = ARG_ANYTHING, -+ }; -+ -+-static u64 bpf_map_delete_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key) -+ { -+- struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; -+- void *key = (void *) (unsigned long) r2; -+- -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+- -+ return map->ops->map_delete_elem(map, key); -+ } -+ -+ const struct bpf_func_proto bpf_map_delete_elem_proto = { -+ .func = bpf_map_delete_elem, -+ .gpl_only = false, -++ .pkt_access = true, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_CONST_MAP_PTR, -+ .arg2_type = ARG_PTR_TO_MAP_KEY, -+ }; -+ -++BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags) -++{ -++ return map->ops->map_push_elem(map, value, flags); -++} -++ -++const struct bpf_func_proto bpf_map_push_elem_proto = { -++ .func = bpf_map_push_elem, -++ .gpl_only = false, -++ .pkt_access = true, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_CONST_MAP_PTR, -++ .arg2_type = ARG_PTR_TO_MAP_VALUE, -++ .arg3_type = ARG_ANYTHING, -++}; -++ -++BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value) -++{ -++ return map->ops->map_pop_elem(map, value); -++} -++ -++const struct bpf_func_proto bpf_map_pop_elem_proto = { -++ .func = bpf_map_pop_elem, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_CONST_MAP_PTR, -++ .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, -++}; -++ -++BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value) -++{ -++ return map->ops->map_peek_elem(map, value); -++} -++ -++const struct bpf_func_proto bpf_map_peek_elem_proto = { -++ .func = bpf_map_peek_elem, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_CONST_MAP_PTR, -++ .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE, -++}; -++ -+ const struct bpf_func_proto bpf_get_prandom_u32_proto = { -+ .func = bpf_user_rnd_u32, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+ }; -+ -+-static u64 bpf_get_smp_processor_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_0(bpf_get_smp_processor_id) -+ { -+- return raw_smp_processor_id(); -++ return smp_processor_id(); -+ } -+ -+ const struct bpf_func_proto bpf_get_smp_processor_id_proto = { -+@@ -110,7 +129,18 @@ const struct bpf_func_proto bpf_get_smp_ -+ .ret_type = RET_INTEGER, -+ }; -+ -+-static u64 bpf_ktime_get_ns(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_0(bpf_get_numa_node_id) -++{ -++ return numa_node_id(); -++} -++ -++const struct bpf_func_proto bpf_get_numa_node_id_proto = { -++ .func = bpf_get_numa_node_id, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++}; -++ -++BPF_CALL_0(bpf_ktime_get_ns) -+ { -+ /* NMI safe access to clock monotonic */ -+ return ktime_get_mono_fast_ns(); -+@@ -122,11 +152,11 @@ const struct bpf_func_proto bpf_ktime_ge -+ .ret_type = RET_INTEGER, -+ }; -+ -+-static u64 bpf_get_current_pid_tgid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_0(bpf_get_current_pid_tgid) -+ { -+ struct task_struct *task = current; -+ -+- if (!task) -++ if (unlikely(!task)) -+ return -EINVAL; -+ -+ return (u64) task->tgid << 32 | task->pid; -+@@ -138,18 +168,18 @@ const struct bpf_func_proto bpf_get_curr -+ .ret_type = RET_INTEGER, -+ }; -+ -+-static u64 bpf_get_current_uid_gid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_0(bpf_get_current_uid_gid) -+ { -+ struct task_struct *task = current; -+ kuid_t uid; -+ kgid_t gid; -+ -+- if (!task) -++ if (unlikely(!task)) -+ return -EINVAL; -+ -+ current_uid_gid(&uid, &gid); -+ return (u64) from_kgid(&init_user_ns, gid) << 32 | -+- from_kuid(&init_user_ns, uid); -++ from_kuid(&init_user_ns, uid); -+ } -+ -+ const struct bpf_func_proto bpf_get_current_uid_gid_proto = { -+@@ -158,22 +188,254 @@ const struct bpf_func_proto bpf_get_curr -+ .ret_type = RET_INTEGER, -+ }; -+ -+-static u64 bpf_get_current_comm(u64 r1, u64 size, u64 r3, u64 r4, u64 r5) -++BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size) -+ { -+ struct task_struct *task = current; -+- char *buf = (char *) (long) r1; -+ -+- if (!task) -+- return -EINVAL; -++ if (unlikely(!task)) -++ goto err_clear; -++ -++ strncpy(buf, task->comm, size); -+ -+- strlcpy(buf, task->comm, min_t(size_t, size, sizeof(task->comm))); -++ /* Verifier guarantees that size > 0. For task->comm exceeding -++ * size, guarantee that buf is %NUL-terminated. Unconditionally -++ * done here to save the size test. -++ */ -++ buf[size - 1] = 0; -+ return 0; -++err_clear: -++ memset(buf, 0, size); -++ return -EINVAL; -+ } -+ -+ const struct bpf_func_proto bpf_get_current_comm_proto = { -+ .func = bpf_get_current_comm, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+- .arg1_type = ARG_PTR_TO_STACK, -+- .arg2_type = ARG_CONST_STACK_SIZE, -++ .arg1_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg2_type = ARG_CONST_SIZE, -++}; -++ -++#if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) -++ -++static inline void __bpf_spin_lock(struct bpf_spin_lock *lock) -++{ -++ arch_spinlock_t *l = (void *)lock; -++ union { -++ __u32 val; -++ arch_spinlock_t lock; -++ } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED }; -++ -++ compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0"); -++ BUILD_BUG_ON(sizeof(*l) != sizeof(__u32)); -++ BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32)); -++ arch_spin_lock(l); -++} -++ -++static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock) -++{ -++ arch_spinlock_t *l = (void *)lock; -++ -++ arch_spin_unlock(l); -++} -++ -++#else -++ -++static inline void __bpf_spin_lock(struct bpf_spin_lock *lock) -++{ -++ atomic_t *l = (void *)lock; -++ -++ BUILD_BUG_ON(sizeof(*l) != sizeof(*lock)); -++ do { -++ smp_cond_load_relaxed(&l->counter, !VAL); -++ } while (atomic_xchg(l, 1)); -++} -++ -++static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock) -++{ -++ atomic_t *l = (void *)lock; -++ -++ atomic_set_release(l, 0); -++} -++ -++#endif -++ -++static DEFINE_PER_CPU(unsigned long, irqsave_flags); -++ -++notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock) -++{ -++ unsigned long flags; -++ -++ local_irq_save(flags); -++ __bpf_spin_lock(lock); -++ __this_cpu_write(irqsave_flags, flags); -++ return 0; -++} -++ -++const struct bpf_func_proto bpf_spin_lock_proto = { -++ .func = bpf_spin_lock, -++ .gpl_only = false, -++ .ret_type = RET_VOID, -++ .arg1_type = ARG_PTR_TO_SPIN_LOCK, -++}; -++ -++notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock) -++{ -++ unsigned long flags; -++ -++ flags = __this_cpu_read(irqsave_flags); -++ __bpf_spin_unlock(lock); -++ local_irq_restore(flags); -++ return 0; -++} -++ -++const struct bpf_func_proto bpf_spin_unlock_proto = { -++ .func = bpf_spin_unlock, -++ .gpl_only = false, -++ .ret_type = RET_VOID, -++ .arg1_type = ARG_PTR_TO_SPIN_LOCK, -++}; -++ -++void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, -++ bool lock_src) -++{ -++ struct bpf_spin_lock *lock; -++ -++ if (lock_src) -++ lock = src + map->spin_lock_off; -++ else -++ lock = dst + map->spin_lock_off; -++ preempt_disable(); -++ ____bpf_spin_lock(lock); -++ copy_map_value(map, dst, src); -++ ____bpf_spin_unlock(lock); -++ preempt_enable(); -++} -++ -++#define BPF_STRTOX_BASE_MASK 0x1F -++ -++static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags, -++ unsigned long long *res, bool *is_negative) -++{ -++ unsigned int base = flags & BPF_STRTOX_BASE_MASK; -++ const char *cur_buf = buf; -++ size_t cur_len = buf_len; -++ unsigned int consumed; -++ size_t val_len; -++ char str[64]; -++ -++ if (!buf || !buf_len || !res || !is_negative) -++ return -EINVAL; -++ -++ if (base != 0 && base != 8 && base != 10 && base != 16) -++ return -EINVAL; -++ -++ if (flags & ~BPF_STRTOX_BASE_MASK) -++ return -EINVAL; -++ -++ while (cur_buf < buf + buf_len && isspace(*cur_buf)) -++ ++cur_buf; -++ -++ *is_negative = (cur_buf < buf + buf_len && *cur_buf == '-'); -++ if (*is_negative) -++ ++cur_buf; -++ -++ consumed = cur_buf - buf; -++ cur_len -= consumed; -++ if (!cur_len) -++ return -EINVAL; -++ -++ cur_len = min(cur_len, sizeof(str) - 1); -++ memcpy(str, cur_buf, cur_len); -++ str[cur_len] = '\0'; -++ cur_buf = str; -++ -++ cur_buf = _parse_integer_fixup_radix(cur_buf, &base); -++ val_len = _parse_integer(cur_buf, base, res); -++ -++ if (val_len & KSTRTOX_OVERFLOW) -++ return -ERANGE; -++ -++ if (val_len == 0) -++ return -EINVAL; -++ -++ cur_buf += val_len; -++ consumed += cur_buf - str; -++ -++ return consumed; -++} -++ -++static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags, -++ long long *res) -++{ -++ unsigned long long _res; -++ bool is_negative; -++ int err; -++ -++ err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative); -++ if (err < 0) -++ return err; -++ if (is_negative) { -++ if ((long long)-_res > 0) -++ return -ERANGE; -++ *res = -_res; -++ } else { -++ if ((long long)_res < 0) -++ return -ERANGE; -++ *res = _res; -++ } -++ return err; -++} -++ -++BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags, -++ long *, res) -++{ -++ long long _res; -++ int err; -++ -++ err = __bpf_strtoll(buf, buf_len, flags, &_res); -++ if (err < 0) -++ return err; -++ if (_res != (long)_res) -++ return -ERANGE; -++ *res = _res; -++ return err; -++} -++ -++const struct bpf_func_proto bpf_strtol_proto = { -++ .func = bpf_strtol, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_MEM, -++ .arg2_type = ARG_CONST_SIZE, -++ .arg3_type = ARG_ANYTHING, -++ .arg4_type = ARG_PTR_TO_LONG, -++}; -++ -++BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags, -++ unsigned long *, res) -++{ -++ unsigned long long _res; -++ bool is_negative; -++ int err; -++ -++ err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative); -++ if (err < 0) -++ return err; -++ if (is_negative) -++ return -EINVAL; -++ if (_res != (unsigned long)_res) -++ return -ERANGE; -++ *res = _res; -++ return err; -++} -++ -++const struct bpf_func_proto bpf_strtoul_proto = { -++ .func = bpf_strtoul, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_MEM, -++ .arg2_type = ARG_CONST_SIZE, -++ .arg3_type = ARG_ANYTHING, -++ .arg4_type = ARG_PTR_TO_LONG, -+ }; -+--- a/kernel/bpf/inode.c -++++ b/kernel/bpf/inode.c -+@@ -1,3 +1,4 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* -+ * Minimal file system backend for holding eBPF maps and programs, -+ * used by bpf(2) object pinning. -+@@ -5,21 +6,19 @@ -+ * Authors: -+ * -+ * Daniel Borkmann -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of the GNU General Public License -+- * version 2 as published by the Free Software Foundation. -+ */ -+ -+-#include -++#include -+ #include -+ #include -+ #include -+ #include -+ #include -+ #include -++#include -+ #include -+ #include -++#include -+ -+ enum bpf_type { -+ BPF_TYPE_UNSPEC = 0, -+@@ -87,6 +86,7 @@ static struct inode *bpf_get_inode(struc -+ switch (mode & S_IFMT) { -+ case S_IFDIR: -+ case S_IFREG: -++ case S_IFLNK: -+ break; -+ default: -+ return ERR_PTR(-EINVAL); -+@@ -119,18 +119,20 @@ static int bpf_inode_type(const struct i -+ return 0; -+ } -+ -+-static bool bpf_dname_reserved(const struct dentry *dentry) -++static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode, -++ struct inode *dir) -+ { -+- return strchr(dentry->d_name.name, '.'); -++ d_instantiate(dentry, inode); -++ dget(dentry); -++ -++ dir->i_mtime = CURRENT_TIME; -++ dir->i_ctime = dir->i_mtime; -+ } -+ -+ static int bpf_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) -+ { -+ struct inode *inode; -+ -+- if (bpf_dname_reserved(dentry)) -+- return -EPERM; -+- -+ inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR); -+ if (IS_ERR(inode)) -+ return PTR_ERR(inode); -+@@ -141,30 +143,30 @@ static int bpf_mkdir(struct inode *dir, -+ inc_nlink(inode); -+ inc_nlink(dir); -+ -+- d_instantiate(dentry, inode); -+- dget(dentry); -+- -++ bpf_dentry_finalize(dentry, inode, dir); -+ return 0; -+ } -+ -+-static int bpf_mkobj_ops(struct inode *dir, struct dentry *dentry, -+- umode_t mode, const struct inode_operations *iops) -++static int bpffs_obj_open(struct inode *inode, struct file *file) -+ { -+- struct inode *inode; -++ return -EIO; -++} -+ -+- if (bpf_dname_reserved(dentry)) -+- return -EPERM; -++static const struct file_operations bpffs_obj_fops = { -++ .open = bpffs_obj_open, -++}; -+ -+- inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFREG); -++static int bpf_mkobj_ops(struct inode *dir, struct dentry *dentry, -++ umode_t mode, const struct inode_operations *iops) -++{ -++ struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFREG); -+ if (IS_ERR(inode)) -+ return PTR_ERR(inode); -+ -+ inode->i_op = iops; -+ inode->i_private = dentry->d_fsdata; -+ -+- d_instantiate(dentry, inode); -+- dget(dentry); -+- -++ bpf_dentry_finalize(dentry, inode, dir); -+ return 0; -+ } -+ -+@@ -187,11 +189,48 @@ static int bpf_mkobj(struct inode *dir, -+ } -+ } -+ -++static struct dentry * -++bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) -++{ -++ /* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future -++ * extensions. -++ */ -++ if (strchr(dentry->d_name.name, '.')) -++ return ERR_PTR(-EPERM); -++ -++ return simple_lookup(dir, dentry, flags); -++} -++ -++static int bpf_symlink(struct inode *dir, struct dentry *dentry, -++ const char *target) -++{ -++ char *link = kstrdup(target, GFP_USER | __GFP_NOWARN); -++ struct inode *inode; -++ -++ if (!link) -++ return -ENOMEM; -++ -++ inode = bpf_get_inode(dir->i_sb, dir, S_IRWXUGO | S_IFLNK); -++ if (IS_ERR(inode)) { -++ kfree(link); -++ return PTR_ERR(inode); -++ } -++ -++ inode->i_op = &simple_symlink_inode_operations; -++ inode->i_link = link; -++ -++ bpf_dentry_finalize(dentry, inode, dir); -++ return 0; -++} -++ -+ static const struct inode_operations bpf_dir_iops = { -+- .lookup = simple_lookup, -++ .lookup = bpf_lookup, -+ .mknod = bpf_mkobj, -+ .mkdir = bpf_mkdir, -++ .symlink = bpf_symlink, -+ .rmdir = simple_rmdir, -++ .rename = simple_rename, -++ .link = simple_link, -+ .unlink = simple_unlink, -+ }; -+ -+@@ -256,7 +295,7 @@ out: -+ } -+ -+ static void *bpf_obj_do_get(const struct filename *pathname, -+- enum bpf_type *type) -++ enum bpf_type *type, int flags) -+ { -+ struct inode *inode; -+ struct path path; -+@@ -268,7 +307,7 @@ static void *bpf_obj_do_get(const struct -+ return ERR_PTR(ret); -+ -+ inode = d_backing_inode(path.dentry); -+- ret = inode_permission(inode, MAY_WRITE); -++ ret = inode_permission(inode, ACC_MODE(flags)); -+ if (ret) -+ goto out; -+ -+@@ -287,18 +326,23 @@ out: -+ return ERR_PTR(ret); -+ } -+ -+-int bpf_obj_get_user(const char __user *pathname) -++int bpf_obj_get_user(const char __user *pathname, int flags) -+ { -+ enum bpf_type type = BPF_TYPE_UNSPEC; -+ struct filename *pname; -+ int ret = -ENOENT; -++ int f_flags; -+ void *raw; -+ -++ f_flags = bpf_get_file_flag(flags); -++ if (f_flags < 0) -++ return f_flags; -++ -+ pname = getname(pathname); -+ if (IS_ERR(pname)) -+ return PTR_ERR(pname); -+ -+- raw = bpf_obj_do_get(pname, &type); -++ raw = bpf_obj_do_get(pname, &type, f_flags); -+ if (IS_ERR(raw)) { -+ ret = PTR_ERR(raw); -+ goto out; -+@@ -307,7 +351,7 @@ int bpf_obj_get_user(const char __user * -+ if (type == BPF_TYPE_PROG) -+ ret = bpf_prog_new_fd(raw); -+ else if (type == BPF_TYPE_MAP) -+- ret = bpf_map_new_fd(raw); -++ ret = bpf_map_new_fd(raw, f_flags); -+ else -+ goto out; -+ -+@@ -318,29 +362,131 @@ out: -+ return ret; -+ } -+ -+-static void bpf_evict_inode(struct inode *inode) -++static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type) -+ { -+- enum bpf_type type; -++ struct bpf_prog *prog; -++ int ret = inode_permission(inode, MAY_READ); -++ if (ret) -++ return ERR_PTR(ret); -++ -++ if (inode->i_op == &bpf_map_iops) -++ return ERR_PTR(-EINVAL); -++ if (inode->i_op != &bpf_prog_iops) -++ return ERR_PTR(-EACCES); -+ -+- truncate_inode_pages_final(&inode->i_data); -+- clear_inode(inode); -++ prog = inode->i_private; -+ -++ if (!bpf_prog_get_ok(prog, &type, false)) -++ return ERR_PTR(-EINVAL); -++ -++ return bpf_prog_inc(prog); -++} -++ -++struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type) -++{ -++ struct bpf_prog *prog; -++ struct path path; -++ int ret = kern_path(name, LOOKUP_FOLLOW, &path); -++ if (ret) -++ return ERR_PTR(ret); -++ prog = __get_prog_inode(d_backing_inode(path.dentry), type); -++ if (!IS_ERR(prog)) -++ touch_atime(&path); -++ path_put(&path); -++ return prog; -++} -++EXPORT_SYMBOL(bpf_prog_get_type_path); -++ -++/* -++ * Display the mount options in /proc/mounts. -++ */ -++static int bpf_show_options(struct seq_file *m, struct dentry *root) -++{ -++ umode_t mode = d_inode(root)->i_mode & S_IALLUGO & ~S_ISVTX; -++ -++ if (mode != S_IRWXUGO) -++ seq_printf(m, ",mode=%o", mode); -++ return 0; -++} -++ -++static void bpf_destroy_inode_deferred(struct rcu_head *head) -++{ -++ struct inode *inode = container_of(head, struct inode, i_rcu); -++ enum bpf_type type; -++ -++ if (S_ISLNK(inode->i_mode)) -++ kfree(inode->i_link); -+ if (!bpf_inode_type(inode, &type)) -+ bpf_any_put(inode->i_private, type); -++ free_inode_nonrcu(inode); -++} -++ -++static void bpf_destroy_inode(struct inode *inode) -++{ -++ call_rcu(&inode->i_rcu, bpf_destroy_inode_deferred); -+ } -+ -+ static const struct super_operations bpf_super_ops = { -+ .statfs = simple_statfs, -+ .drop_inode = generic_delete_inode, -+- .evict_inode = bpf_evict_inode, -++ .show_options = bpf_show_options, -++ .destroy_inode = bpf_destroy_inode, -++}; -++ -++enum { -++ OPT_MODE, -++ OPT_ERR, -++}; -++ -++static const match_table_t bpf_mount_tokens = { -++ { OPT_MODE, "mode=%o" }, -++ { OPT_ERR, NULL }, -++}; -++ -++struct bpf_mount_opts { -++ umode_t mode; -+ }; -+ -++static int bpf_parse_options(char *data, struct bpf_mount_opts *opts) -++{ -++ substring_t args[MAX_OPT_ARGS]; -++ int option, token; -++ char *ptr; -++ -++ opts->mode = S_IRWXUGO; -++ -++ while ((ptr = strsep(&data, ",")) != NULL) { -++ if (!*ptr) -++ continue; -++ -++ token = match_token(ptr, bpf_mount_tokens, args); -++ switch (token) { -++ case OPT_MODE: -++ if (match_octal(&args[0], &option)) -++ return -EINVAL; -++ opts->mode = option & S_IALLUGO; -++ break; -++ /* We might like to report bad mount options here, but -++ * traditionally we've ignored all mount options, so we'd -++ * better continue to ignore non-existing options for bpf. -++ */ -++ } -++ } -++ -++ return 0; -++} -++ -+ static int bpf_fill_super(struct super_block *sb, void *data, int silent) -+ { -+ static struct tree_descr bpf_rfiles[] = { { "" } }; -++ struct bpf_mount_opts opts; -+ struct inode *inode; -+ int ret; -+ -++ ret = bpf_parse_options(data, &opts); -++ if (ret) -++ return ret; -++ -+ ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); -+ if (ret) -+ return ret; -+@@ -350,7 +496,7 @@ static int bpf_fill_super(struct super_b -+ inode = sb->s_root->d_inode; -+ inode->i_op = &bpf_dir_iops; -+ inode->i_mode &= ~S_IALLUGO; -+- inode->i_mode |= S_ISVTX | S_IRWXUGO; -++ inode->i_mode |= S_ISVTX | opts.mode; -+ -+ return 0; -+ } -+@@ -368,8 +514,6 @@ static struct file_system_type bpf_fs_ty -+ .kill_sb = kill_litter_super, -+ }; -+ -+-MODULE_ALIAS_FS("bpf"); -+- -+ static int __init bpf_init(void) -+ { -+ int ret; -+--- /dev/null -++++ b/kernel/bpf/local_storage.c -+@@ -0,0 +1,600 @@ -++//SPDX-License-Identifier: GPL-2.0 -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]); -++ -++#ifdef CONFIG_CGROUP_BPF -++ -++#define LOCAL_STORAGE_CREATE_FLAG_MASK \ -++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) -++ -++struct bpf_cgroup_storage_map { -++ struct bpf_map map; -++ -++ spinlock_t lock; -++ struct bpf_prog *prog; -++ struct rb_root root; -++ struct list_head list; -++}; -++ -++static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map) -++{ -++ return container_of(map, struct bpf_cgroup_storage_map, map); -++} -++ -++static int bpf_cgroup_storage_key_cmp( -++ const struct bpf_cgroup_storage_key *key1, -++ const struct bpf_cgroup_storage_key *key2) -++{ -++ if (key1->cgroup_inode_id < key2->cgroup_inode_id) -++ return -1; -++ else if (key1->cgroup_inode_id > key2->cgroup_inode_id) -++ return 1; -++ else if (key1->attach_type < key2->attach_type) -++ return -1; -++ else if (key1->attach_type > key2->attach_type) -++ return 1; -++ return 0; -++} -++ -++static struct bpf_cgroup_storage *cgroup_storage_lookup( -++ struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key, -++ bool locked) -++{ -++ struct rb_root *root = &map->root; -++ struct rb_node *node; -++ -++ if (!locked) -++ spin_lock_bh(&map->lock); -++ -++ node = root->rb_node; -++ while (node) { -++ struct bpf_cgroup_storage *storage; -++ -++ storage = container_of(node, struct bpf_cgroup_storage, node); -++ -++ switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) { -++ case -1: -++ node = node->rb_left; -++ break; -++ case 1: -++ node = node->rb_right; -++ break; -++ default: -++ if (!locked) -++ spin_unlock_bh(&map->lock); -++ return storage; -++ } -++ } -++ -++ if (!locked) -++ spin_unlock_bh(&map->lock); -++ -++ return NULL; -++} -++ -++static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map, -++ struct bpf_cgroup_storage *storage) -++{ -++ struct rb_root *root = &map->root; -++ struct rb_node **new = &(root->rb_node), *parent = NULL; -++ -++ while (*new) { -++ struct bpf_cgroup_storage *this; -++ -++ this = container_of(*new, struct bpf_cgroup_storage, node); -++ -++ parent = *new; -++ switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) { -++ case -1: -++ new = &((*new)->rb_left); -++ break; -++ case 1: -++ new = &((*new)->rb_right); -++ break; -++ default: -++ return -EEXIST; -++ } -++ } -++ -++ rb_link_node(&storage->node, parent, new); -++ rb_insert_color(&storage->node, root); -++ -++ return 0; -++} -++ -++static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key) -++{ -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage *storage; -++ -++ storage = cgroup_storage_lookup(map, key, false); -++ if (!storage) -++ return NULL; -++ -++ return &READ_ONCE(storage->buf)->data[0]; -++} -++ -++static int cgroup_storage_update_elem(struct bpf_map *map, void *_key, -++ void *value, u64 flags) -++{ -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage *storage; -++ struct bpf_storage_buffer *new; -++ -++ if (unlikely(flags & ~(BPF_F_LOCK | BPF_EXIST | BPF_NOEXIST))) -++ return -EINVAL; -++ -++ if (unlikely(flags & BPF_NOEXIST)) -++ return -EINVAL; -++ -++ if (unlikely((flags & BPF_F_LOCK) && -++ !map_value_has_spin_lock(map))) -++ return -EINVAL; -++ -++ storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map, -++ key, false); -++ if (!storage) -++ return -ENOENT; -++ -++ if (flags & BPF_F_LOCK) { -++ copy_map_value_locked(map, storage->buf->data, value, false); -++ return 0; -++ } -++ -++ new = kmalloc_node(sizeof(struct bpf_storage_buffer) + -++ map->value_size, -++ __GFP_ZERO | GFP_ATOMIC | __GFP_NOWARN, -++ map->numa_node); -++ if (!new) -++ return -ENOMEM; -++ -++ memcpy(&new->data[0], value, map->value_size); -++ check_and_init_map_lock(map, new->data); -++ -++ new = xchg(&storage->buf, new); -++ kfree_rcu(new, rcu); -++ -++ return 0; -++} -++ -++int bpf_percpu_cgroup_storage_copy(struct bpf_map *_map, void *_key, -++ void *value) -++{ -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage *storage; -++ int cpu, off = 0; -++ u32 size; -++ -++ rcu_read_lock(); -++ storage = cgroup_storage_lookup(map, key, false); -++ if (!storage) { -++ rcu_read_unlock(); -++ return -ENOENT; -++ } -++ -++ /* per_cpu areas are zero-filled and bpf programs can only -++ * access 'value_size' of them, so copying rounded areas -++ * will not leak any kernel data -++ */ -++ size = round_up(_map->value_size, 8); -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(value + off, -++ per_cpu_ptr(storage->percpu_buf, cpu), size); -++ off += size; -++ } -++ rcu_read_unlock(); -++ return 0; -++} -++ -++int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *_key, -++ void *value, u64 map_flags) -++{ -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage *storage; -++ int cpu, off = 0; -++ u32 size; -++ -++ if (map_flags != BPF_ANY && map_flags != BPF_EXIST) -++ return -EINVAL; -++ -++ rcu_read_lock(); -++ storage = cgroup_storage_lookup(map, key, false); -++ if (!storage) { -++ rcu_read_unlock(); -++ return -ENOENT; -++ } -++ -++ /* the user space will provide round_up(value_size, 8) bytes that -++ * will be copied into per-cpu area. bpf programs can only access -++ * value_size of it. During lookup the same extra bytes will be -++ * returned or zeros which were zero-filled by percpu_alloc, -++ * so no kernel data leaks possible -++ */ -++ size = round_up(_map->value_size, 8); -++ for_each_possible_cpu(cpu) { -++ bpf_long_memcpy(per_cpu_ptr(storage->percpu_buf, cpu), -++ value + off, size); -++ off += size; -++ } -++ rcu_read_unlock(); -++ return 0; -++} -++ -++static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key, -++ void *_next_key) -++{ -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage_key *next = _next_key; -++ struct bpf_cgroup_storage *storage; -++ -++ spin_lock_bh(&map->lock); -++ -++ if (list_empty(&map->list)) -++ goto enoent; -++ -++ if (key) { -++ storage = cgroup_storage_lookup(map, key, true); -++ if (!storage) -++ goto enoent; -++ -++ storage = list_next_entry(storage, list); -++ if (!storage) -++ goto enoent; -++ } else { -++ storage = list_first_entry(&map->list, -++ struct bpf_cgroup_storage, list); -++ } -++ -++ spin_unlock_bh(&map->lock); -++ next->attach_type = storage->key.attach_type; -++ next->cgroup_inode_id = storage->key.cgroup_inode_id; -++ return 0; -++ -++enoent: -++ spin_unlock_bh(&map->lock); -++ return -ENOENT; -++} -++ -++static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) -++{ -++ int numa_node = bpf_map_attr_numa_node(attr); -++ struct bpf_cgroup_storage_map *map; -++ struct bpf_map_memory mem; -++ int ret; -++ -++ if (attr->key_size != sizeof(struct bpf_cgroup_storage_key)) -++ return ERR_PTR(-EINVAL); -++ -++ if (attr->value_size == 0) -++ return ERR_PTR(-EINVAL); -++ -++ if (attr->value_size > PAGE_SIZE) -++ return ERR_PTR(-E2BIG); -++ -++ if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK || -++ !bpf_map_flags_access_ok(attr->map_flags)) -++ return ERR_PTR(-EINVAL); -++ -++ if (attr->max_entries) -++ /* max_entries is not used and enforced to be 0 */ -++ return ERR_PTR(-EINVAL); -++ -++ ret = bpf_map_charge_init(&mem, sizeof(struct bpf_cgroup_storage_map)); -++ if (ret < 0) -++ return ERR_PTR(ret); -++ -++ map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map), -++ __GFP_ZERO | GFP_USER, numa_node); -++ if (!map) { -++ bpf_map_charge_finish(&mem); -++ return ERR_PTR(-ENOMEM); -++ } -++ -++ bpf_map_charge_move(&map->map.memory, &mem); -++ -++ /* copy mandatory map attributes */ -++ bpf_map_init_from_attr(&map->map, attr); -++ -++ spin_lock_init(&map->lock); -++ map->root = RB_ROOT; -++ INIT_LIST_HEAD(&map->list); -++ -++ return &map->map; -++} -++ -++static void cgroup_storage_map_free(struct bpf_map *_map) -++{ -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ -++ WARN_ON(!RB_EMPTY_ROOT(&map->root)); -++ WARN_ON(!list_empty(&map->list)); -++ -++ kfree(map); -++} -++ -++static int cgroup_storage_delete_elem(struct bpf_map *map, void *key) -++{ -++ return -EINVAL; -++} -++ -++static int cgroup_storage_check_btf(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type) -++{ -++ struct btf_member *m; -++ u32 offset, size; -++ -++ /* Key is expected to be of struct bpf_cgroup_storage_key type, -++ * which is: -++ * struct bpf_cgroup_storage_key { -++ * __u64 cgroup_inode_id; -++ * __u32 attach_type; -++ * }; -++ */ -++ -++ /* -++ * Key_type must be a structure with two fields. -++ */ -++ if (BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT || -++ BTF_INFO_VLEN(key_type->info) != 2) -++ return -EINVAL; -++ -++ /* -++ * The first field must be a 64 bit integer at 0 offset. -++ */ -++ m = (struct btf_member *)(key_type + 1); -++ size = FIELD_SIZEOF(struct bpf_cgroup_storage_key, cgroup_inode_id); -++ if (!btf_member_is_reg_int(btf, key_type, m, 0, size)) -++ return -EINVAL; -++ -++ /* -++ * The second field must be a 32 bit integer at 64 bit offset. -++ */ -++ m++; -++ offset = offsetof(struct bpf_cgroup_storage_key, attach_type); -++ size = FIELD_SIZEOF(struct bpf_cgroup_storage_key, attach_type); -++ if (!btf_member_is_reg_int(btf, key_type, m, offset, size)) -++ return -EINVAL; -++ -++ return 0; -++} -++ -++static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *_key, -++ struct seq_file *m) -++{ -++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(map); -++ struct bpf_cgroup_storage_key *key = _key; -++ struct bpf_cgroup_storage *storage; -++ int cpu; -++ -++ rcu_read_lock(); -++ storage = cgroup_storage_lookup(map_to_storage(map), key, false); -++ if (!storage) { -++ rcu_read_unlock(); -++ return; -++ } -++ -++ btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); -++ stype = cgroup_storage_type(map); -++ if (stype == BPF_CGROUP_STORAGE_SHARED) { -++ seq_puts(m, ": "); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, -++ &READ_ONCE(storage->buf)->data[0], m); -++ seq_puts(m, "\n"); -++ } else { -++ seq_puts(m, ": {\n"); -++ for_each_possible_cpu(cpu) { -++ seq_printf(m, "\tcpu%d: ", cpu); -++ btf_type_seq_show(map->btf, map->btf_value_type_id, -++ per_cpu_ptr(storage->percpu_buf, cpu), -++ m); -++ seq_puts(m, "\n"); -++ } -++ seq_puts(m, "}\n"); -++ } -++ rcu_read_unlock(); -++} -++ -++const struct bpf_map_ops cgroup_storage_map_ops = { -++ .map_alloc = cgroup_storage_map_alloc, -++ .map_free = cgroup_storage_map_free, -++ .map_get_next_key = cgroup_storage_get_next_key, -++ .map_lookup_elem = cgroup_storage_lookup_elem, -++ .map_update_elem = cgroup_storage_update_elem, -++ .map_delete_elem = cgroup_storage_delete_elem, -++ .map_check_btf = cgroup_storage_check_btf, -++ .map_seq_show_elem = cgroup_storage_seq_show_elem, -++}; -++ -++int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map) -++{ -++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ int ret = -EBUSY; -++ -++ spin_lock_bh(&map->lock); -++ -++ if (map->prog && map->prog != prog) -++ goto unlock; -++ if (prog->aux->cgroup_storage[stype] && -++ prog->aux->cgroup_storage[stype] != _map) -++ goto unlock; -++ -++ map->prog = prog; -++ prog->aux->cgroup_storage[stype] = _map; -++ ret = 0; -++unlock: -++ spin_unlock_bh(&map->lock); -++ -++ return ret; -++} -++ -++void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map) -++{ -++ enum bpf_cgroup_storage_type stype = cgroup_storage_type(_map); -++ struct bpf_cgroup_storage_map *map = map_to_storage(_map); -++ -++ spin_lock_bh(&map->lock); -++ if (map->prog == prog) { -++ WARN_ON(prog->aux->cgroup_storage[stype] != _map); -++ map->prog = NULL; -++ prog->aux->cgroup_storage[stype] = NULL; -++ } -++ spin_unlock_bh(&map->lock); -++} -++ -++static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages) -++{ -++ size_t size; -++ -++ if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) { -++ size = sizeof(struct bpf_storage_buffer) + map->value_size; -++ *pages = round_up(sizeof(struct bpf_cgroup_storage) + size, -++ PAGE_SIZE) >> PAGE_SHIFT; -++ } else { -++ size = map->value_size; -++ *pages = round_up(round_up(size, 8) * num_possible_cpus(), -++ PAGE_SIZE) >> PAGE_SHIFT; -++ } -++ -++ return size; -++} -++ -++struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, -++ enum bpf_cgroup_storage_type stype) -++{ -++ struct bpf_cgroup_storage *storage; -++ struct bpf_map *map; -++ gfp_t flags; -++ size_t size; -++ u32 pages; -++ -++ map = prog->aux->cgroup_storage[stype]; -++ if (!map) -++ return NULL; -++ -++ size = bpf_cgroup_storage_calculate_size(map, &pages); -++ -++ if (bpf_map_charge_memlock(map, pages)) -++ return ERR_PTR(-EPERM); -++ -++ storage = kmalloc_node(sizeof(struct bpf_cgroup_storage), -++ __GFP_ZERO | GFP_USER, map->numa_node); -++ if (!storage) -++ goto enomem; -++ -++ flags = __GFP_ZERO | GFP_USER; -++ -++ if (stype == BPF_CGROUP_STORAGE_SHARED) { -++ storage->buf = kmalloc_node(size, flags, map->numa_node); -++ if (!storage->buf) -++ goto enomem; -++ check_and_init_map_lock(map, storage->buf->data); -++ } else { -++ storage->percpu_buf = __alloc_percpu_gfp(size, 8, flags); -++ if (!storage->percpu_buf) -++ goto enomem; -++ } -++ -++ storage->map = (struct bpf_cgroup_storage_map *)map; -++ -++ return storage; -++ -++enomem: -++ bpf_map_uncharge_memlock(map, pages); -++ kfree(storage); -++ return ERR_PTR(-ENOMEM); -++} -++ -++static void free_shared_cgroup_storage_rcu(struct rcu_head *rcu) -++{ -++ struct bpf_cgroup_storage *storage = -++ container_of(rcu, struct bpf_cgroup_storage, rcu); -++ -++ kfree(storage->buf); -++ kfree(storage); -++} -++ -++static void free_percpu_cgroup_storage_rcu(struct rcu_head *rcu) -++{ -++ struct bpf_cgroup_storage *storage = -++ container_of(rcu, struct bpf_cgroup_storage, rcu); -++ -++ free_percpu(storage->percpu_buf); -++ kfree(storage); -++} -++ -++void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage) -++{ -++ enum bpf_cgroup_storage_type stype; -++ struct bpf_map *map; -++ u32 pages; -++ -++ if (!storage) -++ return; -++ -++ map = &storage->map->map; -++ -++ bpf_cgroup_storage_calculate_size(map, &pages); -++ bpf_map_uncharge_memlock(map, pages); -++ -++ stype = cgroup_storage_type(map); -++ if (stype == BPF_CGROUP_STORAGE_SHARED) -++ call_rcu(&storage->rcu, free_shared_cgroup_storage_rcu); -++ else -++ call_rcu(&storage->rcu, free_percpu_cgroup_storage_rcu); -++} -++ -++void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, -++ struct cgroup *cgroup, -++ enum bpf_attach_type type) -++{ -++ struct bpf_cgroup_storage_map *map; -++ -++ if (!storage) -++ return; -++ -++ storage->key.attach_type = type; -++ storage->key.cgroup_inode_id = cgroup->kn->id.id; -++ -++ map = storage->map; -++ -++ spin_lock_bh(&map->lock); -++ WARN_ON(cgroup_storage_insert(map, storage)); -++ list_add(&storage->list, &map->list); -++ spin_unlock_bh(&map->lock); -++} -++ -++void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage) -++{ -++ struct bpf_cgroup_storage_map *map; -++ struct rb_root *root; -++ -++ if (!storage) -++ return; -++ -++ map = storage->map; -++ -++ spin_lock_bh(&map->lock); -++ root = &map->root; -++ rb_erase(&storage->node, root); -++ -++ list_del(&storage->list); -++ spin_unlock_bh(&map->lock); -++} -++ -++#endif -+--- /dev/null -++++ b/kernel/bpf/lpm_trie.c -+@@ -0,0 +1,746 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* -++ * Longest prefix match list implementation -++ * -++ * Copyright (c) 2016,2017 Daniel Mack -++ * Copyright (c) 2016 David Herrmann -++ */ -++ -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++/* Intermediate node */ -++#define LPM_TREE_NODE_FLAG_IM BIT(0) -++ -++struct lpm_trie_node; -++ -++struct lpm_trie_node { -++ struct rcu_head rcu; -++ struct lpm_trie_node __rcu *child[2]; -++ u32 prefixlen; -++ u32 flags; -++ u8 data[0]; -++}; -++ -++struct lpm_trie { -++ struct bpf_map map; -++ struct lpm_trie_node __rcu *root; -++ size_t n_entries; -++ size_t max_prefixlen; -++ size_t data_size; -++ raw_spinlock_t lock; -++}; -++ -++/* This trie implements a longest prefix match algorithm that can be used to -++ * match IP addresses to a stored set of ranges. -++ * -++ * Data stored in @data of struct bpf_lpm_key and struct lpm_trie_node is -++ * interpreted as big endian, so data[0] stores the most significant byte. -++ * -++ * Match ranges are internally stored in instances of struct lpm_trie_node -++ * which each contain their prefix length as well as two pointers that may -++ * lead to more nodes containing more specific matches. Each node also stores -++ * a value that is defined by and returned to userspace via the update_elem -++ * and lookup functions. -++ * -++ * For instance, let's start with a trie that was created with a prefix length -++ * of 32, so it can be used for IPv4 addresses, and one single element that -++ * matches 192.168.0.0/16. The data array would hence contain -++ * [0xc0, 0xa8, 0x00, 0x00] in big-endian notation. This documentation will -++ * stick to IP-address notation for readability though. -++ * -++ * As the trie is empty initially, the new node (1) will be places as root -++ * node, denoted as (R) in the example below. As there are no other node, both -++ * child pointers are %NULL. -++ * -++ * +----------------+ -++ * | (1) (R) | -++ * | 192.168.0.0/16 | -++ * | value: 1 | -++ * | [0] [1] | -++ * +----------------+ -++ * -++ * Next, let's add a new node (2) matching 192.168.0.0/24. As there is already -++ * a node with the same data and a smaller prefix (ie, a less specific one), -++ * node (2) will become a child of (1). In child index depends on the next bit -++ * that is outside of what (1) matches, and that bit is 0, so (2) will be -++ * child[0] of (1): -++ * -++ * +----------------+ -++ * | (1) (R) | -++ * | 192.168.0.0/16 | -++ * | value: 1 | -++ * | [0] [1] | -++ * +----------------+ -++ * | -++ * +----------------+ -++ * | (2) | -++ * | 192.168.0.0/24 | -++ * | value: 2 | -++ * | [0] [1] | -++ * +----------------+ -++ * -++ * The child[1] slot of (1) could be filled with another node which has bit #17 -++ * (the next bit after the ones that (1) matches on) set to 1. For instance, -++ * 192.168.128.0/24: -++ * -++ * +----------------+ -++ * | (1) (R) | -++ * | 192.168.0.0/16 | -++ * | value: 1 | -++ * | [0] [1] | -++ * +----------------+ -++ * | | -++ * +----------------+ +------------------+ -++ * | (2) | | (3) | -++ * | 192.168.0.0/24 | | 192.168.128.0/24 | -++ * | value: 2 | | value: 3 | -++ * | [0] [1] | | [0] [1] | -++ * +----------------+ +------------------+ -++ * -++ * Let's add another node (4) to the game for 192.168.1.0/24. In order to place -++ * it, node (1) is looked at first, and because (4) of the semantics laid out -++ * above (bit #17 is 0), it would normally be attached to (1) as child[0]. -++ * However, that slot is already allocated, so a new node is needed in between. -++ * That node does not have a value attached to it and it will never be -++ * returned to users as result of a lookup. It is only there to differentiate -++ * the traversal further. It will get a prefix as wide as necessary to -++ * distinguish its two children: -++ * -++ * +----------------+ -++ * | (1) (R) | -++ * | 192.168.0.0/16 | -++ * | value: 1 | -++ * | [0] [1] | -++ * +----------------+ -++ * | | -++ * +----------------+ +------------------+ -++ * | (4) (I) | | (3) | -++ * | 192.168.0.0/23 | | 192.168.128.0/24 | -++ * | value: --- | | value: 3 | -++ * | [0] [1] | | [0] [1] | -++ * +----------------+ +------------------+ -++ * | | -++ * +----------------+ +----------------+ -++ * | (2) | | (5) | -++ * | 192.168.0.0/24 | | 192.168.1.0/24 | -++ * | value: 2 | | value: 5 | -++ * | [0] [1] | | [0] [1] | -++ * +----------------+ +----------------+ -++ * -++ * 192.168.1.1/32 would be a child of (5) etc. -++ * -++ * An intermediate node will be turned into a 'real' node on demand. In the -++ * example above, (4) would be re-used if 192.168.0.0/23 is added to the trie. -++ * -++ * A fully populated trie would have a height of 32 nodes, as the trie was -++ * created with a prefix length of 32. -++ * -++ * The lookup starts at the root node. If the current node matches and if there -++ * is a child that can be used to become more specific, the trie is traversed -++ * downwards. The last node in the traversal that is a non-intermediate one is -++ * returned. -++ */ -++ -++static inline int extract_bit(const u8 *data, size_t index) -++{ -++ return !!(data[index / 8] & (1 << (7 - (index % 8)))); -++} -++ -++/** -++ * longest_prefix_match() - determine the longest prefix -++ * @trie: The trie to get internal sizes from -++ * @node: The node to operate on -++ * @key: The key to compare to @node -++ * -++ * Determine the longest prefix of @node that matches the bits in @key. -++ */ -++static size_t longest_prefix_match(const struct lpm_trie *trie, -++ const struct lpm_trie_node *node, -++ const struct bpf_lpm_trie_key *key) -++{ -++ u32 limit = min(node->prefixlen, key->prefixlen); -++ u32 prefixlen = 0, i = 0; -++ -++ BUILD_BUG_ON(offsetof(struct lpm_trie_node, data) % sizeof(u32)); -++ BUILD_BUG_ON(offsetof(struct bpf_lpm_trie_key, data) % sizeof(u32)); -++ -++#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && defined(CONFIG_64BIT) -++ -++ /* data_size >= 16 has very small probability. -++ * We do not use a loop for optimal code generation. -++ */ -++ if (trie->data_size >= 8) { -++ u64 diff = be64_to_cpu(*(__be64 *)node->data ^ -++ *(__be64 *)key->data); -++ -++ prefixlen = 64 - fls64(diff); -++ if (prefixlen >= limit) -++ return limit; -++ if (diff) -++ return prefixlen; -++ i = 8; -++ } -++#endif -++ -++ while (trie->data_size >= i + 4) { -++ u32 diff = be32_to_cpu(*(__be32 *)&node->data[i] ^ -++ *(__be32 *)&key->data[i]); -++ -++ prefixlen += 32 - fls(diff); -++ if (prefixlen >= limit) -++ return limit; -++ if (diff) -++ return prefixlen; -++ i += 4; -++ } -++ -++ if (trie->data_size >= i + 2) { -++ u16 diff = be16_to_cpu(*(__be16 *)&node->data[i] ^ -++ *(__be16 *)&key->data[i]); -++ -++ prefixlen += 16 - fls(diff); -++ if (prefixlen >= limit) -++ return limit; -++ if (diff) -++ return prefixlen; -++ i += 2; -++ } -++ -++ if (trie->data_size >= i + 1) { -++ prefixlen += 8 - fls(node->data[i] ^ key->data[i]); -++ -++ if (prefixlen >= limit) -++ return limit; -++ } -++ -++ return prefixlen; -++} -++ -++/* Called from syscall or from eBPF program */ -++static void *trie_lookup_elem(struct bpf_map *map, void *_key) -++{ -++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); -++ struct lpm_trie_node *node, *found = NULL; -++ struct bpf_lpm_trie_key *key = _key; -++ -++ /* Start walking the trie from the root node ... */ -++ -++ for (node = rcu_dereference(trie->root); node;) { -++ unsigned int next_bit; -++ size_t matchlen; -++ -++ /* Determine the longest prefix of @node that matches @key. -++ * If it's the maximum possible prefix for this trie, we have -++ * an exact match and can return it directly. -++ */ -++ matchlen = longest_prefix_match(trie, node, key); -++ if (matchlen == trie->max_prefixlen) { -++ found = node; -++ break; -++ } -++ -++ /* If the number of bits that match is smaller than the prefix -++ * length of @node, bail out and return the node we have seen -++ * last in the traversal (ie, the parent). -++ */ -++ if (matchlen < node->prefixlen) -++ break; -++ -++ /* Consider this node as return candidate unless it is an -++ * artificially added intermediate one. -++ */ -++ if (!(node->flags & LPM_TREE_NODE_FLAG_IM)) -++ found = node; -++ -++ /* If the node match is fully satisfied, let's see if we can -++ * become more specific. Determine the next bit in the key and -++ * traverse down. -++ */ -++ next_bit = extract_bit(key->data, node->prefixlen); -++ node = rcu_dereference(node->child[next_bit]); -++ } -++ -++ if (!found) -++ return NULL; -++ -++ return found->data + trie->data_size; -++} -++ -++static struct lpm_trie_node *lpm_trie_node_alloc(const struct lpm_trie *trie, -++ const void *value) -++{ -++ struct lpm_trie_node *node; -++ size_t size = sizeof(struct lpm_trie_node) + trie->data_size; -++ -++ if (value) -++ size += trie->map.value_size; -++ -++ node = kmalloc_node(size, GFP_ATOMIC | __GFP_NOWARN, -++ trie->map.numa_node); -++ if (!node) -++ return NULL; -++ -++ node->flags = 0; -++ -++ if (value) -++ memcpy(node->data + trie->data_size, value, -++ trie->map.value_size); -++ -++ return node; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int trie_update_elem(struct bpf_map *map, -++ void *_key, void *value, u64 flags) -++{ -++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); -++ struct lpm_trie_node *node, *im_node = NULL, *new_node = NULL; -++ struct lpm_trie_node __rcu **slot; -++ struct bpf_lpm_trie_key *key = _key; -++ unsigned long irq_flags; -++ unsigned int next_bit; -++ size_t matchlen = 0; -++ int ret = 0; -++ -++ if (unlikely(flags > BPF_EXIST)) -++ return -EINVAL; -++ -++ if (key->prefixlen > trie->max_prefixlen) -++ return -EINVAL; -++ -++ raw_spin_lock_irqsave(&trie->lock, irq_flags); -++ -++ /* Allocate and fill a new node */ -++ -++ if (trie->n_entries == trie->map.max_entries) { -++ ret = -ENOSPC; -++ goto out; -++ } -++ -++ new_node = lpm_trie_node_alloc(trie, value); -++ if (!new_node) { -++ ret = -ENOMEM; -++ goto out; -++ } -++ -++ trie->n_entries++; -++ -++ new_node->prefixlen = key->prefixlen; -++ RCU_INIT_POINTER(new_node->child[0], NULL); -++ RCU_INIT_POINTER(new_node->child[1], NULL); -++ memcpy(new_node->data, key->data, trie->data_size); -++ -++ /* Now find a slot to attach the new node. To do that, walk the tree -++ * from the root and match as many bits as possible for each node until -++ * we either find an empty slot or a slot that needs to be replaced by -++ * an intermediate node. -++ */ -++ slot = &trie->root; -++ -++ while ((node = rcu_dereference_protected(*slot, -++ lockdep_is_held(&trie->lock)))) { -++ matchlen = longest_prefix_match(trie, node, key); -++ -++ if (node->prefixlen != matchlen || -++ node->prefixlen == key->prefixlen || -++ node->prefixlen == trie->max_prefixlen) -++ break; -++ -++ next_bit = extract_bit(key->data, node->prefixlen); -++ slot = &node->child[next_bit]; -++ } -++ -++ /* If the slot is empty (a free child pointer or an empty root), -++ * simply assign the @new_node to that slot and be done. -++ */ -++ if (!node) { -++ rcu_assign_pointer(*slot, new_node); -++ goto out; -++ } -++ -++ /* If the slot we picked already exists, replace it with @new_node -++ * which already has the correct data array set. -++ */ -++ if (node->prefixlen == matchlen) { -++ new_node->child[0] = node->child[0]; -++ new_node->child[1] = node->child[1]; -++ -++ if (!(node->flags & LPM_TREE_NODE_FLAG_IM)) -++ trie->n_entries--; -++ -++ rcu_assign_pointer(*slot, new_node); -++ kfree_rcu(node, rcu); -++ -++ goto out; -++ } -++ -++ /* If the new node matches the prefix completely, it must be inserted -++ * as an ancestor. Simply insert it between @node and *@slot. -++ */ -++ if (matchlen == key->prefixlen) { -++ next_bit = extract_bit(node->data, matchlen); -++ rcu_assign_pointer(new_node->child[next_bit], node); -++ rcu_assign_pointer(*slot, new_node); -++ goto out; -++ } -++ -++ im_node = lpm_trie_node_alloc(trie, NULL); -++ if (!im_node) { -++ ret = -ENOMEM; -++ goto out; -++ } -++ -++ im_node->prefixlen = matchlen; -++ im_node->flags |= LPM_TREE_NODE_FLAG_IM; -++ memcpy(im_node->data, node->data, trie->data_size); -++ -++ /* Now determine which child to install in which slot */ -++ if (extract_bit(key->data, matchlen)) { -++ rcu_assign_pointer(im_node->child[0], node); -++ rcu_assign_pointer(im_node->child[1], new_node); -++ } else { -++ rcu_assign_pointer(im_node->child[0], new_node); -++ rcu_assign_pointer(im_node->child[1], node); -++ } -++ -++ /* Finally, assign the intermediate node to the determined spot */ -++ rcu_assign_pointer(*slot, im_node); -++ -++out: -++ if (ret) { -++ if (new_node) -++ trie->n_entries--; -++ -++ kfree(new_node); -++ kfree(im_node); -++ } -++ -++ raw_spin_unlock_irqrestore(&trie->lock, irq_flags); -++ -++ return ret; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int trie_delete_elem(struct bpf_map *map, void *_key) -++{ -++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); -++ struct bpf_lpm_trie_key *key = _key; -++ struct lpm_trie_node __rcu **trim, **trim2; -++ struct lpm_trie_node *node, *parent; -++ unsigned long irq_flags; -++ unsigned int next_bit; -++ size_t matchlen = 0; -++ int ret = 0; -++ -++ if (key->prefixlen > trie->max_prefixlen) -++ return -EINVAL; -++ -++ raw_spin_lock_irqsave(&trie->lock, irq_flags); -++ -++ /* Walk the tree looking for an exact key/length match and keeping -++ * track of the path we traverse. We will need to know the node -++ * we wish to delete, and the slot that points to the node we want -++ * to delete. We may also need to know the nodes parent and the -++ * slot that contains it. -++ */ -++ trim = &trie->root; -++ trim2 = trim; -++ parent = NULL; -++ while ((node = rcu_dereference_protected( -++ *trim, lockdep_is_held(&trie->lock)))) { -++ matchlen = longest_prefix_match(trie, node, key); -++ -++ if (node->prefixlen != matchlen || -++ node->prefixlen == key->prefixlen) -++ break; -++ -++ parent = node; -++ trim2 = trim; -++ next_bit = extract_bit(key->data, node->prefixlen); -++ trim = &node->child[next_bit]; -++ } -++ -++ if (!node || node->prefixlen != key->prefixlen || -++ node->prefixlen != matchlen || -++ (node->flags & LPM_TREE_NODE_FLAG_IM)) { -++ ret = -ENOENT; -++ goto out; -++ } -++ -++ trie->n_entries--; -++ -++ /* If the node we are removing has two children, simply mark it -++ * as intermediate and we are done. -++ */ -++ if (rcu_access_pointer(node->child[0]) && -++ rcu_access_pointer(node->child[1])) { -++ node->flags |= LPM_TREE_NODE_FLAG_IM; -++ goto out; -++ } -++ -++ /* If the parent of the node we are about to delete is an intermediate -++ * node, and the deleted node doesn't have any children, we can delete -++ * the intermediate parent as well and promote its other child -++ * up the tree. Doing this maintains the invariant that all -++ * intermediate nodes have exactly 2 children and that there are no -++ * unnecessary intermediate nodes in the tree. -++ */ -++ if (parent && (parent->flags & LPM_TREE_NODE_FLAG_IM) && -++ !node->child[0] && !node->child[1]) { -++ if (node == rcu_access_pointer(parent->child[0])) -++ rcu_assign_pointer( -++ *trim2, rcu_access_pointer(parent->child[1])); -++ else -++ rcu_assign_pointer( -++ *trim2, rcu_access_pointer(parent->child[0])); -++ kfree_rcu(parent, rcu); -++ kfree_rcu(node, rcu); -++ goto out; -++ } -++ -++ /* The node we are removing has either zero or one child. If there -++ * is a child, move it into the removed node's slot then delete -++ * the node. Otherwise just clear the slot and delete the node. -++ */ -++ if (node->child[0]) -++ rcu_assign_pointer(*trim, rcu_access_pointer(node->child[0])); -++ else if (node->child[1]) -++ rcu_assign_pointer(*trim, rcu_access_pointer(node->child[1])); -++ else -++ RCU_INIT_POINTER(*trim, NULL); -++ kfree_rcu(node, rcu); -++ -++out: -++ raw_spin_unlock_irqrestore(&trie->lock, irq_flags); -++ -++ return ret; -++} -++ -++#define LPM_DATA_SIZE_MAX 256 -++#define LPM_DATA_SIZE_MIN 1 -++ -++#define LPM_VAL_SIZE_MAX (KMALLOC_MAX_SIZE - LPM_DATA_SIZE_MAX - \ -++ sizeof(struct lpm_trie_node)) -++#define LPM_VAL_SIZE_MIN 1 -++ -++#define LPM_KEY_SIZE(X) (sizeof(struct bpf_lpm_trie_key) + (X)) -++#define LPM_KEY_SIZE_MAX LPM_KEY_SIZE(LPM_DATA_SIZE_MAX) -++#define LPM_KEY_SIZE_MIN LPM_KEY_SIZE(LPM_DATA_SIZE_MIN) -++ -++#define LPM_CREATE_FLAG_MASK (BPF_F_NO_PREALLOC | BPF_F_NUMA_NODE | \ -++ BPF_F_ACCESS_MASK) -++ -++static struct bpf_map *trie_alloc(union bpf_attr *attr) -++{ -++ struct lpm_trie *trie; -++ u64 cost = sizeof(*trie), cost_per_node; -++ int ret; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return ERR_PTR(-EPERM); -++ -++ /* check sanity of attributes */ -++ if (attr->max_entries == 0 || -++ !(attr->map_flags & BPF_F_NO_PREALLOC) || -++ attr->map_flags & ~LPM_CREATE_FLAG_MASK || -++ !bpf_map_flags_access_ok(attr->map_flags) || -++ attr->key_size < LPM_KEY_SIZE_MIN || -++ attr->key_size > LPM_KEY_SIZE_MAX || -++ attr->value_size < LPM_VAL_SIZE_MIN || -++ attr->value_size > LPM_VAL_SIZE_MAX) -++ return ERR_PTR(-EINVAL); -++ -++ trie = kzalloc(sizeof(*trie), GFP_USER | __GFP_NOWARN); -++ if (!trie) -++ return ERR_PTR(-ENOMEM); -++ -++ /* copy mandatory map attributes */ -++ bpf_map_init_from_attr(&trie->map, attr); -++ trie->data_size = attr->key_size - -++ offsetof(struct bpf_lpm_trie_key, data); -++ trie->max_prefixlen = trie->data_size * 8; -++ -++ cost_per_node = sizeof(struct lpm_trie_node) + -++ attr->value_size + trie->data_size; -++ cost += (u64) attr->max_entries * cost_per_node; -++ -++ ret = bpf_map_charge_init(&trie->map.memory, cost); -++ if (ret) -++ goto out_err; -++ -++ raw_spin_lock_init(&trie->lock); -++ -++ return &trie->map; -++out_err: -++ kfree(trie); -++ return ERR_PTR(ret); -++} -++ -++static void trie_free(struct bpf_map *map) -++{ -++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); -++ struct lpm_trie_node __rcu **slot; -++ struct lpm_trie_node *node; -++ -++ /* Wait for outstanding programs to complete -++ * update/lookup/delete/get_next_key and free the trie. -++ */ -++ synchronize_rcu(); -++ -++ /* Always start at the root and walk down to a node that has no -++ * children. Then free that node, nullify its reference in the parent -++ * and start over. -++ */ -++ -++ for (;;) { -++ slot = &trie->root; -++ -++ for (;;) { -++ node = rcu_dereference_protected(*slot, 1); -++ if (!node) -++ goto out; -++ -++ if (rcu_access_pointer(node->child[0])) { -++ slot = &node->child[0]; -++ continue; -++ } -++ -++ if (rcu_access_pointer(node->child[1])) { -++ slot = &node->child[1]; -++ continue; -++ } -++ -++ kfree(node); -++ RCU_INIT_POINTER(*slot, NULL); -++ break; -++ } -++ } -++ -++out: -++ kfree(trie); -++} -++ -++static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key) -++{ -++ struct lpm_trie_node *node, *next_node = NULL, *parent, *search_root; -++ struct lpm_trie *trie = container_of(map, struct lpm_trie, map); -++ struct bpf_lpm_trie_key *key = _key, *next_key = _next_key; -++ struct lpm_trie_node **node_stack = NULL; -++ int err = 0, stack_ptr = -1; -++ unsigned int next_bit; -++ size_t matchlen; -++ -++ /* The get_next_key follows postorder. For the 4 node example in -++ * the top of this file, the trie_get_next_key() returns the following -++ * one after another: -++ * 192.168.0.0/24 -++ * 192.168.1.0/24 -++ * 192.168.128.0/24 -++ * 192.168.0.0/16 -++ * -++ * The idea is to return more specific keys before less specific ones. -++ */ -++ -++ /* Empty trie */ -++ search_root = rcu_dereference(trie->root); -++ if (!search_root) -++ return -ENOENT; -++ -++ /* For invalid key, find the leftmost node in the trie */ -++ if (!key || key->prefixlen > trie->max_prefixlen) -++ goto find_leftmost; -++ -++ node_stack = kmalloc_array(trie->max_prefixlen, -++ sizeof(struct lpm_trie_node *), -++ GFP_ATOMIC | __GFP_NOWARN); -++ if (!node_stack) -++ return -ENOMEM; -++ -++ /* Try to find the exact node for the given key */ -++ for (node = search_root; node;) { -++ node_stack[++stack_ptr] = node; -++ matchlen = longest_prefix_match(trie, node, key); -++ if (node->prefixlen != matchlen || -++ node->prefixlen == key->prefixlen) -++ break; -++ -++ next_bit = extract_bit(key->data, node->prefixlen); -++ node = rcu_dereference(node->child[next_bit]); -++ } -++ if (!node || node->prefixlen != key->prefixlen || -++ (node->flags & LPM_TREE_NODE_FLAG_IM)) -++ goto find_leftmost; -++ -++ /* The node with the exactly-matching key has been found, -++ * find the first node in postorder after the matched node. -++ */ -++ node = node_stack[stack_ptr]; -++ while (stack_ptr > 0) { -++ parent = node_stack[stack_ptr - 1]; -++ if (rcu_dereference(parent->child[0]) == node) { -++ search_root = rcu_dereference(parent->child[1]); -++ if (search_root) -++ goto find_leftmost; -++ } -++ if (!(parent->flags & LPM_TREE_NODE_FLAG_IM)) { -++ next_node = parent; -++ goto do_copy; -++ } -++ -++ node = parent; -++ stack_ptr--; -++ } -++ -++ /* did not find anything */ -++ err = -ENOENT; -++ goto free_stack; -++ -++find_leftmost: -++ /* Find the leftmost non-intermediate node, all intermediate nodes -++ * have exact two children, so this function will never return NULL. -++ */ -++ for (node = search_root; node;) { -++ if (node->flags & LPM_TREE_NODE_FLAG_IM) { -++ node = rcu_dereference(node->child[0]); -++ } else { -++ next_node = node; -++ node = rcu_dereference(node->child[0]); -++ if (!node) -++ node = rcu_dereference(next_node->child[1]); -++ } -++ } -++do_copy: -++ next_key->prefixlen = next_node->prefixlen; -++ memcpy((void *)next_key + offsetof(struct bpf_lpm_trie_key, data), -++ next_node->data, trie->data_size); -++free_stack: -++ kfree(node_stack); -++ return err; -++} -++ -++static int trie_check_btf(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type) -++{ -++ /* Keys must have struct bpf_lpm_trie_key embedded. */ -++ return BTF_INFO_KIND(key_type->info) != BTF_KIND_STRUCT ? -++ -EINVAL : 0; -++} -++ -++const struct bpf_map_ops trie_map_ops = { -++ .map_alloc = trie_alloc, -++ .map_free = trie_free, -++ .map_get_next_key = trie_get_next_key, -++ .map_lookup_elem = trie_lookup_elem, -++ .map_update_elem = trie_update_elem, -++ .map_delete_elem = trie_delete_elem, -++ .map_check_btf = trie_check_btf, -++}; -+--- a/kernel/bpf/Makefile -++++ b/kernel/bpf/Makefile -+@@ -1,4 +1,23 @@ -++# SPDX-License-Identifier: GPL-2.0 -+ obj-y := core.o -++ifneq ($(CONFIG_BPF_JIT_ALWAYS_ON),y) -++# ___bpf_prog_run() needs GCSE disabled on x86; see 3193c0836f203 for details -++cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse -++endif -++CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy) -+ -+-obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o -+-obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o -++obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o -++obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o -++obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o -++obj-$(CONFIG_BPF_SYSCALL) += disasm.o -++obj-$(CONFIG_BPF_SYSCALL) += btf.o -++ifeq ($(CONFIG_NET),y) -++obj-$(CONFIG_BPF_SYSCALL) += devmap.o -++endif -++ifeq ($(CONFIG_PERF_EVENTS),y) -++obj-$(CONFIG_BPF_SYSCALL) += stackmap.o -++endif -++obj-$(CONFIG_CGROUP_BPF) += cgroup.o -++ifeq ($(CONFIG_SYSFS),y) -++obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o -++endif -+--- /dev/null -++++ b/kernel/bpf/map_in_map.c -+@@ -0,0 +1,120 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2017 Facebook -++ */ -++#include -++#include -++ -++#include "map_in_map.h" -++ -++struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) -++{ -++ struct bpf_map *inner_map, *inner_map_meta; -++ u32 inner_map_meta_size; -++ struct fd f; -++ -++ f = fdget(inner_map_ufd); -++ inner_map = __bpf_map_get(f); -++ if (IS_ERR(inner_map)) -++ return inner_map; -++ -++ /* prog_array->owner_prog_type and owner_jited -++ * is a runtime binding. Doing static check alone -++ * in the verifier is not enough. -++ */ -++ if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY || -++ inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || -++ inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) { -++ fdput(f); -++ return ERR_PTR(-ENOTSUPP); -++ } -++ -++ /* Does not support >1 level map-in-map */ -++ if (inner_map->inner_map_meta) { -++ fdput(f); -++ return ERR_PTR(-EINVAL); -++ } -++ -++ if (map_value_has_spin_lock(inner_map)) { -++ fdput(f); -++ return ERR_PTR(-ENOTSUPP); -++ } -++ -++ inner_map_meta_size = sizeof(*inner_map_meta); -++ /* In some cases verifier needs to access beyond just base map. */ -++ if (inner_map->ops == &array_map_ops) -++ inner_map_meta_size = sizeof(struct bpf_array); -++ -++ inner_map_meta = kzalloc(inner_map_meta_size, GFP_USER); -++ if (!inner_map_meta) { -++ fdput(f); -++ return ERR_PTR(-ENOMEM); -++ } -++ -++ inner_map_meta->map_type = inner_map->map_type; -++ inner_map_meta->key_size = inner_map->key_size; -++ inner_map_meta->value_size = inner_map->value_size; -++ inner_map_meta->map_flags = inner_map->map_flags; -++ inner_map_meta->max_entries = inner_map->max_entries; -++ inner_map_meta->spin_lock_off = inner_map->spin_lock_off; -++ -++ /* Misc members not needed in bpf_map_meta_equal() check. */ -++ inner_map_meta->ops = inner_map->ops; -++ if (inner_map->ops == &array_map_ops) { -++ inner_map_meta->unpriv_array = inner_map->unpriv_array; -++ container_of(inner_map_meta, struct bpf_array, map)->index_mask = -++ container_of(inner_map, struct bpf_array, map)->index_mask; -++ } -++ -++ fdput(f); -++ return inner_map_meta; -++} -++ -++void bpf_map_meta_free(struct bpf_map *map_meta) -++{ -++ kfree(map_meta); -++} -++ -++bool bpf_map_meta_equal(const struct bpf_map *meta0, -++ const struct bpf_map *meta1) -++{ -++ /* No need to compare ops because it is covered by map_type */ -++ return meta0->map_type == meta1->map_type && -++ meta0->key_size == meta1->key_size && -++ meta0->value_size == meta1->value_size && -++ meta0->map_flags == meta1->map_flags && -++ meta0->max_entries == meta1->max_entries; -++} -++ -++void *bpf_map_fd_get_ptr(struct bpf_map *map, -++ struct file *map_file /* not used */, -++ int ufd) -++{ -++ struct bpf_map *inner_map; -++ struct fd f; -++ -++ f = fdget(ufd); -++ inner_map = __bpf_map_get(f); -++ if (IS_ERR(inner_map)) -++ return inner_map; -++ -++ if (bpf_map_meta_equal(map->inner_map_meta, inner_map)) -++ inner_map = bpf_map_inc(inner_map, false); -++ else -++ inner_map = ERR_PTR(-EINVAL); -++ -++ fdput(f); -++ return inner_map; -++} -++ -++void bpf_map_fd_put_ptr(void *ptr) -++{ -++ /* ptr->ops->map_free() has to go through one -++ * rcu grace period by itself. -++ */ -++ bpf_map_put(ptr); -++} -++ -++u32 bpf_map_fd_sys_lookup_elem(void *ptr) -++{ -++ return ((struct bpf_map *)ptr)->id; -++} -+--- /dev/null -++++ b/kernel/bpf/map_in_map.h -+@@ -0,0 +1,21 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* Copyright (c) 2017 Facebook -++ */ -++#ifndef __MAP_IN_MAP_H__ -++#define __MAP_IN_MAP_H__ -++ -++#include -++ -++struct file; -++struct bpf_map; -++ -++struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd); -++void bpf_map_meta_free(struct bpf_map *map_meta); -++bool bpf_map_meta_equal(const struct bpf_map *meta0, -++ const struct bpf_map *meta1); -++void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file, -++ int ufd); -++void bpf_map_fd_put_ptr(void *ptr); -++u32 bpf_map_fd_sys_lookup_elem(void *ptr); -++ -++#endif -+--- /dev/null -++++ b/kernel/bpf/percpu_freelist.c -+@@ -0,0 +1,118 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2016 Facebook -++ */ -++#include "percpu_freelist.h" -++ -++int pcpu_freelist_init(struct pcpu_freelist *s) -++{ -++ int cpu; -++ -++ s->freelist = alloc_percpu(struct pcpu_freelist_head); -++ if (!s->freelist) -++ return -ENOMEM; -++ -++ for_each_possible_cpu(cpu) { -++ struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu); -++ -++ raw_spin_lock_init(&head->lock); -++ head->first = NULL; -++ } -++ return 0; -++} -++ -++void pcpu_freelist_destroy(struct pcpu_freelist *s) -++{ -++ free_percpu(s->freelist); -++} -++ -++static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, -++ struct pcpu_freelist_node *node) -++{ -++ raw_spin_lock(&head->lock); -++ node->next = head->first; -++ head->first = node; -++ raw_spin_unlock(&head->lock); -++} -++ -++void __pcpu_freelist_push(struct pcpu_freelist *s, -++ struct pcpu_freelist_node *node) -++{ -++ struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist); -++ -++ ___pcpu_freelist_push(head, node); -++} -++ -++void pcpu_freelist_push(struct pcpu_freelist *s, -++ struct pcpu_freelist_node *node) -++{ -++ unsigned long flags; -++ -++ local_irq_save(flags); -++ __pcpu_freelist_push(s, node); -++ local_irq_restore(flags); -++} -++ -++void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size, -++ u32 nr_elems) -++{ -++ struct pcpu_freelist_head *head; -++ unsigned long flags; -++ int i, cpu, pcpu_entries; -++ -++ pcpu_entries = nr_elems / num_possible_cpus() + 1; -++ i = 0; -++ -++ /* disable irq to workaround lockdep false positive -++ * in bpf usage pcpu_freelist_populate() will never race -++ * with pcpu_freelist_push() -++ */ -++ local_irq_save(flags); -++ for_each_possible_cpu(cpu) { -++again: -++ head = per_cpu_ptr(s->freelist, cpu); -++ ___pcpu_freelist_push(head, buf); -++ i++; -++ buf += elem_size; -++ if (i == nr_elems) -++ break; -++ if (i % pcpu_entries) -++ goto again; -++ } -++ local_irq_restore(flags); -++} -++ -++struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s) -++{ -++ struct pcpu_freelist_head *head; -++ struct pcpu_freelist_node *node; -++ int orig_cpu, cpu; -++ -++ orig_cpu = cpu = raw_smp_processor_id(); -++ while (1) { -++ head = per_cpu_ptr(s->freelist, cpu); -++ raw_spin_lock(&head->lock); -++ node = head->first; -++ if (node) { -++ head->first = node->next; -++ raw_spin_unlock(&head->lock); -++ return node; -++ } -++ raw_spin_unlock(&head->lock); -++ cpu = cpumask_next(cpu, cpu_possible_mask); -++ if (cpu >= nr_cpu_ids) -++ cpu = 0; -++ if (cpu == orig_cpu) -++ return NULL; -++ } -++} -++ -++struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s) -++{ -++ struct pcpu_freelist_node *ret; -++ unsigned long flags; -++ -++ local_irq_save(flags); -++ ret = __pcpu_freelist_pop(s); -++ local_irq_restore(flags); -++ return ret; -++} -+--- /dev/null -++++ b/kernel/bpf/percpu_freelist.h -+@@ -0,0 +1,32 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* Copyright (c) 2016 Facebook -++ */ -++#ifndef __PERCPU_FREELIST_H__ -++#define __PERCPU_FREELIST_H__ -++#include -++#include -++ -++struct pcpu_freelist_head { -++ struct pcpu_freelist_node *first; -++ raw_spinlock_t lock; -++}; -++ -++struct pcpu_freelist { -++ struct pcpu_freelist_head __percpu *freelist; -++}; -++ -++struct pcpu_freelist_node { -++ struct pcpu_freelist_node *next; -++}; -++ -++/* pcpu_freelist_* do spin_lock_irqsave. */ -++void pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *); -++struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *); -++/* __pcpu_freelist_* do spin_lock only. caller must disable irqs. */ -++void __pcpu_freelist_push(struct pcpu_freelist *, struct pcpu_freelist_node *); -++struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *); -++void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size, -++ u32 nr_elems); -++int pcpu_freelist_init(struct pcpu_freelist *); -++void pcpu_freelist_destroy(struct pcpu_freelist *s); -++#endif -+--- /dev/null -++++ b/kernel/bpf/queue_stack_maps.c -+@@ -0,0 +1,289 @@ -++// SPDX-License-Identifier: GPL-2.0 -++/* -++ * queue_stack_maps.c: BPF queue and stack maps -++ * -++ * Copyright (c) 2018 Politecnico di Torino -++ */ -++#include -++#include -++#include -++#include -++#include "percpu_freelist.h" -++ -++#define QUEUE_STACK_CREATE_FLAG_MASK \ -++ (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK) -++ -++struct bpf_queue_stack { -++ struct bpf_map map; -++ raw_spinlock_t lock; -++ u32 head, tail; -++ u32 size; /* max_entries + 1 */ -++ -++ char elements[0] __aligned(8); -++}; -++ -++static struct bpf_queue_stack *bpf_queue_stack(struct bpf_map *map) -++{ -++ return container_of(map, struct bpf_queue_stack, map); -++} -++ -++static bool queue_stack_map_is_empty(struct bpf_queue_stack *qs) -++{ -++ return qs->head == qs->tail; -++} -++ -++static bool queue_stack_map_is_full(struct bpf_queue_stack *qs) -++{ -++ u32 head = qs->head + 1; -++ -++ if (unlikely(head >= qs->size)) -++ head = 0; -++ -++ return head == qs->tail; -++} -++ -++/* Called from syscall */ -++static int queue_stack_map_alloc_check(union bpf_attr *attr) -++{ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ /* check sanity of attributes */ -++ if (attr->max_entries == 0 || attr->key_size != 0 || -++ attr->value_size == 0 || -++ attr->map_flags & ~QUEUE_STACK_CREATE_FLAG_MASK || -++ !bpf_map_flags_access_ok(attr->map_flags)) -++ return -EINVAL; -++ -++ if (attr->value_size > KMALLOC_MAX_SIZE) -++ /* if value_size is bigger, the user space won't be able to -++ * access the elements. -++ */ -++ return -E2BIG; -++ -++ return 0; -++} -++ -++static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr) -++{ -++ int ret, numa_node = bpf_map_attr_numa_node(attr); -++ struct bpf_map_memory mem = {0}; -++ struct bpf_queue_stack *qs; -++ u64 size, queue_size, cost; -++ -++ size = (u64) attr->max_entries + 1; -++ cost = queue_size = sizeof(*qs) + size * attr->value_size; -++ -++ ret = bpf_map_charge_init(&mem, cost); -++ if (ret < 0) -++ return ERR_PTR(ret); -++ -++ qs = bpf_map_area_alloc(queue_size, numa_node); -++ if (!qs) { -++ bpf_map_charge_finish(&mem); -++ return ERR_PTR(-ENOMEM); -++ } -++ -++ memset(qs, 0, sizeof(*qs)); -++ -++ bpf_map_init_from_attr(&qs->map, attr); -++ -++ bpf_map_charge_move(&qs->map.memory, &mem); -++ qs->size = size; -++ -++ raw_spin_lock_init(&qs->lock); -++ -++ return &qs->map; -++} -++ -++/* Called when map->refcnt goes to zero, either from workqueue or from syscall */ -++static void queue_stack_map_free(struct bpf_map *map) -++{ -++ struct bpf_queue_stack *qs = bpf_queue_stack(map); -++ -++ /* at this point bpf_prog->aux->refcnt == 0 and this map->refcnt == 0, -++ * so the programs (can be more than one that used this map) were -++ * disconnected from events. Wait for outstanding critical sections in -++ * these programs to complete -++ */ -++ synchronize_rcu(); -++ -++ bpf_map_area_free(qs); -++} -++ -++static int __queue_map_get(struct bpf_map *map, void *value, bool delete) -++{ -++ struct bpf_queue_stack *qs = bpf_queue_stack(map); -++ unsigned long flags; -++ int err = 0; -++ void *ptr; -++ -++ raw_spin_lock_irqsave(&qs->lock, flags); -++ -++ if (queue_stack_map_is_empty(qs)) { -++ memset(value, 0, qs->map.value_size); -++ err = -ENOENT; -++ goto out; -++ } -++ -++ ptr = &qs->elements[qs->tail * qs->map.value_size]; -++ memcpy(value, ptr, qs->map.value_size); -++ -++ if (delete) { -++ if (unlikely(++qs->tail >= qs->size)) -++ qs->tail = 0; -++ } -++ -++out: -++ raw_spin_unlock_irqrestore(&qs->lock, flags); -++ return err; -++} -++ -++ -++static int __stack_map_get(struct bpf_map *map, void *value, bool delete) -++{ -++ struct bpf_queue_stack *qs = bpf_queue_stack(map); -++ unsigned long flags; -++ int err = 0; -++ void *ptr; -++ u32 index; -++ -++ raw_spin_lock_irqsave(&qs->lock, flags); -++ -++ if (queue_stack_map_is_empty(qs)) { -++ memset(value, 0, qs->map.value_size); -++ err = -ENOENT; -++ goto out; -++ } -++ -++ index = qs->head - 1; -++ if (unlikely(index >= qs->size)) -++ index = qs->size - 1; -++ -++ ptr = &qs->elements[index * qs->map.value_size]; -++ memcpy(value, ptr, qs->map.value_size); -++ -++ if (delete) -++ qs->head = index; -++ -++out: -++ raw_spin_unlock_irqrestore(&qs->lock, flags); -++ return err; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int queue_map_peek_elem(struct bpf_map *map, void *value) -++{ -++ return __queue_map_get(map, value, false); -++} -++ -++/* Called from syscall or from eBPF program */ -++static int stack_map_peek_elem(struct bpf_map *map, void *value) -++{ -++ return __stack_map_get(map, value, false); -++} -++ -++/* Called from syscall or from eBPF program */ -++static int queue_map_pop_elem(struct bpf_map *map, void *value) -++{ -++ return __queue_map_get(map, value, true); -++} -++ -++/* Called from syscall or from eBPF program */ -++static int stack_map_pop_elem(struct bpf_map *map, void *value) -++{ -++ return __stack_map_get(map, value, true); -++} -++ -++/* Called from syscall or from eBPF program */ -++static int queue_stack_map_push_elem(struct bpf_map *map, void *value, -++ u64 flags) -++{ -++ struct bpf_queue_stack *qs = bpf_queue_stack(map); -++ unsigned long irq_flags; -++ int err = 0; -++ void *dst; -++ -++ /* BPF_EXIST is used to force making room for a new element in case the -++ * map is full -++ */ -++ bool replace = (flags & BPF_EXIST); -++ -++ /* Check supported flags for queue and stack maps */ -++ if (flags & BPF_NOEXIST || flags > BPF_EXIST) -++ return -EINVAL; -++ -++ raw_spin_lock_irqsave(&qs->lock, irq_flags); -++ -++ if (queue_stack_map_is_full(qs)) { -++ if (!replace) { -++ err = -E2BIG; -++ goto out; -++ } -++ /* advance tail pointer to overwrite oldest element */ -++ if (unlikely(++qs->tail >= qs->size)) -++ qs->tail = 0; -++ } -++ -++ dst = &qs->elements[qs->head * qs->map.value_size]; -++ memcpy(dst, value, qs->map.value_size); -++ -++ if (unlikely(++qs->head >= qs->size)) -++ qs->head = 0; -++ -++out: -++ raw_spin_unlock_irqrestore(&qs->lock, irq_flags); -++ return err; -++} -++ -++/* Called from syscall or from eBPF program */ -++static void *queue_stack_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ return NULL; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int queue_stack_map_update_elem(struct bpf_map *map, void *key, -++ void *value, u64 flags) -++{ -++ return -EINVAL; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int queue_stack_map_delete_elem(struct bpf_map *map, void *key) -++{ -++ return -EINVAL; -++} -++ -++/* Called from syscall */ -++static int queue_stack_map_get_next_key(struct bpf_map *map, void *key, -++ void *next_key) -++{ -++ return -EINVAL; -++} -++ -++const struct bpf_map_ops queue_map_ops = { -++ .map_alloc_check = queue_stack_map_alloc_check, -++ .map_alloc = queue_stack_map_alloc, -++ .map_free = queue_stack_map_free, -++ .map_lookup_elem = queue_stack_map_lookup_elem, -++ .map_update_elem = queue_stack_map_update_elem, -++ .map_delete_elem = queue_stack_map_delete_elem, -++ .map_push_elem = queue_stack_map_push_elem, -++ .map_pop_elem = queue_map_pop_elem, -++ .map_peek_elem = queue_map_peek_elem, -++ .map_get_next_key = queue_stack_map_get_next_key, -++}; -++ -++const struct bpf_map_ops stack_map_ops = { -++ .map_alloc_check = queue_stack_map_alloc_check, -++ .map_alloc = queue_stack_map_alloc, -++ .map_free = queue_stack_map_free, -++ .map_lookup_elem = queue_stack_map_lookup_elem, -++ .map_update_elem = queue_stack_map_update_elem, -++ .map_delete_elem = queue_stack_map_delete_elem, -++ .map_push_elem = queue_stack_map_push_elem, -++ .map_pop_elem = stack_map_pop_elem, -++ .map_peek_elem = stack_map_peek_elem, -++ .map_get_next_key = queue_stack_map_get_next_key, -++}; -+--- /dev/null -++++ b/kernel/bpf/stackmap.c -+@@ -0,0 +1,634 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* Copyright (c) 2016 Facebook -++ */ -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include "percpu_freelist.h" -++ -++#define STACK_CREATE_FLAG_MASK \ -++ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \ -++ BPF_F_STACK_BUILD_ID) -++ -++struct stack_map_bucket { -++ struct pcpu_freelist_node fnode; -++ u32 hash; -++ u32 nr; -++ u64 data[]; -++}; -++ -++struct bpf_stack_map { -++ struct bpf_map map; -++ void *elems; -++ struct pcpu_freelist freelist; -++ u32 n_buckets; -++ struct stack_map_bucket *buckets[]; -++}; -++ -++/* irq_work to run up_read() for build_id lookup in nmi context */ -++struct stack_map_irq_work { -++ struct irq_work irq_work; -++ struct rw_semaphore *sem; -++}; -++ -++static void do_up_read(struct irq_work *entry) -++{ -++ struct stack_map_irq_work *work; -++ -++ work = container_of(entry, struct stack_map_irq_work, irq_work); -++ up_read_non_owner(work->sem); -++ work->sem = NULL; -++} -++ -++static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work); -++ -++static inline bool stack_map_use_build_id(struct bpf_map *map) -++{ -++ return (map->map_flags & BPF_F_STACK_BUILD_ID); -++} -++ -++static inline int stack_map_data_size(struct bpf_map *map) -++{ -++ return stack_map_use_build_id(map) ? -++ sizeof(struct bpf_stack_build_id) : sizeof(u64); -++} -++ -++static int prealloc_elems_and_freelist(struct bpf_stack_map *smap) -++{ -++ u32 elem_size = sizeof(struct stack_map_bucket) + smap->map.value_size; -++ int err; -++ -++ smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries, -++ smap->map.numa_node); -++ if (!smap->elems) -++ return -ENOMEM; -++ -++ err = pcpu_freelist_init(&smap->freelist); -++ if (err) -++ goto free_elems; -++ -++ pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size, -++ smap->map.max_entries); -++ return 0; -++ -++free_elems: -++ bpf_map_area_free(smap->elems); -++ return err; -++} -++ -++/* Called from syscall */ -++static struct bpf_map *stack_map_alloc(union bpf_attr *attr) -++{ -++ u32 value_size = attr->value_size; -++ struct bpf_stack_map *smap; -++ struct bpf_map_memory mem; -++ u64 cost, n_buckets; -++ int err; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return ERR_PTR(-EPERM); -++ -++ if (attr->map_flags & ~STACK_CREATE_FLAG_MASK) -++ return ERR_PTR(-EINVAL); -++ -++ /* check sanity of attributes */ -++ if (attr->max_entries == 0 || attr->key_size != 4 || -++ value_size < 8 || value_size % 8) -++ return ERR_PTR(-EINVAL); -++ -++ BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64)); -++ if (attr->map_flags & BPF_F_STACK_BUILD_ID) { -++ if (value_size % sizeof(struct bpf_stack_build_id) || -++ value_size / sizeof(struct bpf_stack_build_id) -++ > sysctl_perf_event_max_stack) -++ return ERR_PTR(-EINVAL); -++ } else if (value_size / 8 > sysctl_perf_event_max_stack) -++ return ERR_PTR(-EINVAL); -++ -++ /* hash table size must be power of 2 */ -++ n_buckets = roundup_pow_of_two(attr->max_entries); -++ if (!n_buckets) -++ return ERR_PTR(-E2BIG); -++ -++ cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap); -++ cost += n_buckets * (value_size + sizeof(struct stack_map_bucket)); -++ err = bpf_map_charge_init(&mem, cost); -++ if (err) -++ return ERR_PTR(err); -++ -++ smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr)); -++ if (!smap) { -++ bpf_map_charge_finish(&mem); -++ return ERR_PTR(-ENOMEM); -++ } -++ -++ bpf_map_init_from_attr(&smap->map, attr); -++ smap->map.value_size = value_size; -++ smap->n_buckets = n_buckets; -++ -++ err = get_callchain_buffers(sysctl_perf_event_max_stack); -++ if (err) -++ goto free_charge; -++ -++ err = prealloc_elems_and_freelist(smap); -++ if (err) -++ goto put_buffers; -++ -++ bpf_map_charge_move(&smap->map.memory, &mem); -++ -++ return &smap->map; -++ -++put_buffers: -++ put_callchain_buffers(); -++free_charge: -++ bpf_map_charge_finish(&mem); -++ bpf_map_area_free(smap); -++ return ERR_PTR(err); -++} -++ -++#define BPF_BUILD_ID 3 -++/* -++ * Parse build id from the note segment. This logic can be shared between -++ * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are -++ * identical. -++ */ -++static inline int stack_map_parse_build_id(void *page_addr, -++ unsigned char *build_id, -++ void *note_start, -++ Elf32_Word note_size) -++{ -++ Elf32_Word note_offs = 0, new_offs; -++ -++ /* check for overflow */ -++ if (note_start < page_addr || note_start + note_size < note_start) -++ return -EINVAL; -++ -++ /* only supports note that fits in the first page */ -++ if (note_start + note_size > page_addr + PAGE_SIZE) -++ return -EINVAL; -++ -++ while (note_offs + sizeof(Elf32_Nhdr) < note_size) { -++ Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); -++ -++ if (nhdr->n_type == BPF_BUILD_ID && -++ nhdr->n_namesz == sizeof("GNU") && -++ nhdr->n_descsz > 0 && -++ nhdr->n_descsz <= BPF_BUILD_ID_SIZE) { -++ memcpy(build_id, -++ note_start + note_offs + -++ ALIGN(sizeof("GNU"), 4) + sizeof(Elf32_Nhdr), -++ nhdr->n_descsz); -++ memset(build_id + nhdr->n_descsz, 0, -++ BPF_BUILD_ID_SIZE - nhdr->n_descsz); -++ return 0; -++ } -++ new_offs = note_offs + sizeof(Elf32_Nhdr) + -++ ALIGN(nhdr->n_namesz, 4) + ALIGN(nhdr->n_descsz, 4); -++ if (new_offs <= note_offs) /* overflow */ -++ break; -++ note_offs = new_offs; -++ } -++ return -EINVAL; -++} -++ -++/* Parse build ID from 32-bit ELF */ -++static int stack_map_get_build_id_32(void *page_addr, -++ unsigned char *build_id) -++{ -++ Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; -++ Elf32_Phdr *phdr; -++ int i; -++ -++ /* only supports phdr that fits in one page */ -++ if (ehdr->e_phnum > -++ (PAGE_SIZE - sizeof(Elf32_Ehdr)) / sizeof(Elf32_Phdr)) -++ return -EINVAL; -++ -++ phdr = (Elf32_Phdr *)(page_addr + sizeof(Elf32_Ehdr)); -++ -++ for (i = 0; i < ehdr->e_phnum; ++i) -++ if (phdr[i].p_type == PT_NOTE) -++ return stack_map_parse_build_id(page_addr, build_id, -++ page_addr + phdr[i].p_offset, -++ phdr[i].p_filesz); -++ return -EINVAL; -++} -++ -++/* Parse build ID from 64-bit ELF */ -++static int stack_map_get_build_id_64(void *page_addr, -++ unsigned char *build_id) -++{ -++ Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; -++ Elf64_Phdr *phdr; -++ int i; -++ -++ /* only supports phdr that fits in one page */ -++ if (ehdr->e_phnum > -++ (PAGE_SIZE - sizeof(Elf64_Ehdr)) / sizeof(Elf64_Phdr)) -++ return -EINVAL; -++ -++ phdr = (Elf64_Phdr *)(page_addr + sizeof(Elf64_Ehdr)); -++ -++ for (i = 0; i < ehdr->e_phnum; ++i) -++ if (phdr[i].p_type == PT_NOTE) -++ return stack_map_parse_build_id(page_addr, build_id, -++ page_addr + phdr[i].p_offset, -++ phdr[i].p_filesz); -++ return -EINVAL; -++} -++ -++/* Parse build ID of ELF file mapped to vma */ -++static int stack_map_get_build_id(struct vm_area_struct *vma, -++ unsigned char *build_id) -++{ -++ Elf32_Ehdr *ehdr; -++ struct page *page; -++ void *page_addr; -++ int ret; -++ -++ /* only works for page backed storage */ -++ if (!vma->vm_file) -++ return -EINVAL; -++ -++ page = find_get_page(vma->vm_file->f_mapping, 0); -++ if (!page) -++ return -EFAULT; /* page not mapped */ -++ -++ ret = -EINVAL; -++ page_addr = kmap_atomic(page); -++ ehdr = (Elf32_Ehdr *)page_addr; -++ -++ /* compare magic x7f "ELF" */ -++ if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) -++ goto out; -++ -++ /* only support executable file and shared object file */ -++ if (ehdr->e_type != ET_EXEC && ehdr->e_type != ET_DYN) -++ goto out; -++ -++ if (ehdr->e_ident[EI_CLASS] == ELFCLASS32) -++ ret = stack_map_get_build_id_32(page_addr, build_id); -++ else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64) -++ ret = stack_map_get_build_id_64(page_addr, build_id); -++out: -++ kunmap_atomic(page_addr); -++ put_page(page); -++ return ret; -++} -++ -++static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, -++ u64 *ips, u32 trace_nr, bool user) -++{ -++ int i; -++ struct vm_area_struct *vma; -++ bool irq_work_busy = false; -++ struct stack_map_irq_work *work = NULL; -++ -++ if (irqs_disabled()) { -++ work = this_cpu_ptr(&up_read_work); -++ if (work->irq_work.flags & IRQ_WORK_BUSY) -++ /* cannot queue more up_read, fallback */ -++ irq_work_busy = true; -++ } -++ -++ /* -++ * We cannot do up_read() when the irq is disabled, because of -++ * risk to deadlock with rq_lock. To do build_id lookup when the -++ * irqs are disabled, we need to run up_read() in irq_work. We use -++ * a percpu variable to do the irq_work. If the irq_work is -++ * already used by another lookup, we fall back to report ips. -++ * -++ * Same fallback is used for kernel stack (!user) on a stackmap -++ * with build_id. -++ */ -++ if (!user || !current || !current->mm || irq_work_busy || -++ down_read_trylock(¤t->mm->mmap_sem) == 0) { -++ /* cannot access current->mm, fall back to ips */ -++ for (i = 0; i < trace_nr; i++) { -++ id_offs[i].status = BPF_STACK_BUILD_ID_IP; -++ id_offs[i].ip = ips[i]; -++ memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); -++ } -++ return; -++ } -++ -++ for (i = 0; i < trace_nr; i++) { -++ vma = find_vma(current->mm, ips[i]); -++ if (!vma || stack_map_get_build_id(vma, id_offs[i].build_id)) { -++ /* per entry fall back to ips */ -++ id_offs[i].status = BPF_STACK_BUILD_ID_IP; -++ id_offs[i].ip = ips[i]; -++ memset(id_offs[i].build_id, 0, BPF_BUILD_ID_SIZE); -++ continue; -++ } -++ id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i] -++ - vma->vm_start; -++ id_offs[i].status = BPF_STACK_BUILD_ID_VALID; -++ } -++ -++ if (!work) { -++ up_read(¤t->mm->mmap_sem); -++ } else { -++ work->sem = ¤t->mm->mmap_sem; -++ irq_work_queue(&work->irq_work); -++ /* -++ * The irq_work will release the mmap_sem with -++ * up_read_non_owner(). The rwsem_release() is called -++ * here to release the lock from lockdep's perspective. -++ */ -++ rwsem_release(¤t->mm->mmap_sem.dep_map, 1, _RET_IP_); -++ } -++} -++ -++BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, -++ u64, flags) -++{ -++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); -++ struct perf_callchain_entry *trace; -++ struct stack_map_bucket *bucket, *new_bucket, *old_bucket; -++ u32 max_depth = map->value_size / stack_map_data_size(map); -++ /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */ -++ u32 init_nr = sysctl_perf_event_max_stack - max_depth; -++ u32 skip = flags & BPF_F_SKIP_FIELD_MASK; -++ u32 hash, id, trace_nr, trace_len; -++ bool user = flags & BPF_F_USER_STACK; -++ bool kernel = !user; -++ u64 *ips; -++ bool hash_matches; -++ -++ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | -++ BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) -++ return -EINVAL; -++ -++ trace = get_perf_callchain(regs, init_nr, kernel, user, -++ sysctl_perf_event_max_stack, false, false); -++ -++ if (unlikely(!trace)) -++ /* couldn't fetch the stack trace */ -++ return -EFAULT; -++ -++ /* get_perf_callchain() guarantees that trace->nr >= init_nr -++ * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth -++ */ -++ trace_nr = trace->nr - init_nr; -++ -++ if (trace_nr <= skip) -++ /* skipping more than usable stack trace */ -++ return -EFAULT; -++ -++ trace_nr -= skip; -++ trace_len = trace_nr * sizeof(u64); -++ ips = trace->ip + skip + init_nr; -++ hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); -++ id = hash & (smap->n_buckets - 1); -++ bucket = READ_ONCE(smap->buckets[id]); -++ -++ hash_matches = bucket && bucket->hash == hash; -++ /* fast cmp */ -++ if (hash_matches && flags & BPF_F_FAST_STACK_CMP) -++ return id; -++ -++ if (stack_map_use_build_id(map)) { -++ /* for build_id+offset, pop a bucket before slow cmp */ -++ new_bucket = (struct stack_map_bucket *) -++ pcpu_freelist_pop(&smap->freelist); -++ if (unlikely(!new_bucket)) -++ return -ENOMEM; -++ new_bucket->nr = trace_nr; -++ stack_map_get_build_id_offset( -++ (struct bpf_stack_build_id *)new_bucket->data, -++ ips, trace_nr, user); -++ trace_len = trace_nr * sizeof(struct bpf_stack_build_id); -++ if (hash_matches && bucket->nr == trace_nr && -++ memcmp(bucket->data, new_bucket->data, trace_len) == 0) { -++ pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); -++ return id; -++ } -++ if (bucket && !(flags & BPF_F_REUSE_STACKID)) { -++ pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); -++ return -EEXIST; -++ } -++ } else { -++ if (hash_matches && bucket->nr == trace_nr && -++ memcmp(bucket->data, ips, trace_len) == 0) -++ return id; -++ if (bucket && !(flags & BPF_F_REUSE_STACKID)) -++ return -EEXIST; -++ -++ new_bucket = (struct stack_map_bucket *) -++ pcpu_freelist_pop(&smap->freelist); -++ if (unlikely(!new_bucket)) -++ return -ENOMEM; -++ memcpy(new_bucket->data, ips, trace_len); -++ } -++ -++ new_bucket->hash = hash; -++ new_bucket->nr = trace_nr; -++ -++ old_bucket = xchg(&smap->buckets[id], new_bucket); -++ if (old_bucket) -++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); -++ return id; -++} -++ -++const struct bpf_func_proto bpf_get_stackid_proto = { -++ .func = bpf_get_stackid, -++ .gpl_only = true, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_CONST_MAP_PTR, -++ .arg3_type = ARG_ANYTHING, -++}; -++ -++BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, -++ u64, flags) -++{ -++ u32 init_nr, trace_nr, copy_len, elem_size, num_elem; -++ bool user_build_id = flags & BPF_F_USER_BUILD_ID; -++ u32 skip = flags & BPF_F_SKIP_FIELD_MASK; -++ bool user = flags & BPF_F_USER_STACK; -++ struct perf_callchain_entry *trace; -++ bool kernel = !user; -++ int err = -EINVAL; -++ u64 *ips; -++ -++ if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | -++ BPF_F_USER_BUILD_ID))) -++ goto clear; -++ if (kernel && user_build_id) -++ goto clear; -++ -++ elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id) -++ : sizeof(u64); -++ if (unlikely(size % elem_size)) -++ goto clear; -++ -++ num_elem = size / elem_size; -++ if (sysctl_perf_event_max_stack < num_elem) -++ init_nr = 0; -++ else -++ init_nr = sysctl_perf_event_max_stack - num_elem; -++ trace = get_perf_callchain(regs, init_nr, kernel, user, -++ sysctl_perf_event_max_stack, false, false); -++ if (unlikely(!trace)) -++ goto err_fault; -++ -++ trace_nr = trace->nr - init_nr; -++ if (trace_nr < skip) -++ goto err_fault; -++ -++ trace_nr -= skip; -++ trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem; -++ copy_len = trace_nr * elem_size; -++ ips = trace->ip + skip + init_nr; -++ if (user && user_build_id) -++ stack_map_get_build_id_offset(buf, ips, trace_nr, user); -++ else -++ memcpy(buf, ips, copy_len); -++ -++ if (size > copy_len) -++ memset(buf + copy_len, 0, size - copy_len); -++ return copy_len; -++ -++err_fault: -++ err = -EFAULT; -++clear: -++ memset(buf, 0, size); -++ return err; -++} -++ -++const struct bpf_func_proto bpf_get_stack_proto = { -++ .func = bpf_get_stack, -++ .gpl_only = true, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg3_type = ARG_CONST_SIZE_OR_ZERO, -++ .arg4_type = ARG_ANYTHING, -++}; -++ -++/* Called from eBPF program */ -++static void *stack_map_lookup_elem(struct bpf_map *map, void *key) -++{ -++ return ERR_PTR(-EOPNOTSUPP); -++} -++ -++/* Called from syscall */ -++int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) -++{ -++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); -++ struct stack_map_bucket *bucket, *old_bucket; -++ u32 id = *(u32 *)key, trace_len; -++ -++ if (unlikely(id >= smap->n_buckets)) -++ return -ENOENT; -++ -++ bucket = xchg(&smap->buckets[id], NULL); -++ if (!bucket) -++ return -ENOENT; -++ -++ trace_len = bucket->nr * stack_map_data_size(map); -++ memcpy(value, bucket->data, trace_len); -++ memset(value + trace_len, 0, map->value_size - trace_len); -++ -++ old_bucket = xchg(&smap->buckets[id], bucket); -++ if (old_bucket) -++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); -++ return 0; -++} -++ -++static int stack_map_get_next_key(struct bpf_map *map, void *key, -++ void *next_key) -++{ -++ struct bpf_stack_map *smap = container_of(map, -++ struct bpf_stack_map, map); -++ u32 id; -++ -++ WARN_ON_ONCE(!rcu_read_lock_held()); -++ -++ if (!key) { -++ id = 0; -++ } else { -++ id = *(u32 *)key; -++ if (id >= smap->n_buckets || !smap->buckets[id]) -++ id = 0; -++ else -++ id++; -++ } -++ -++ while (id < smap->n_buckets && !smap->buckets[id]) -++ id++; -++ -++ if (id >= smap->n_buckets) -++ return -ENOENT; -++ -++ *(u32 *)next_key = id; -++ return 0; -++} -++ -++static int stack_map_update_elem(struct bpf_map *map, void *key, void *value, -++ u64 map_flags) -++{ -++ return -EINVAL; -++} -++ -++/* Called from syscall or from eBPF program */ -++static int stack_map_delete_elem(struct bpf_map *map, void *key) -++{ -++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); -++ struct stack_map_bucket *old_bucket; -++ u32 id = *(u32 *)key; -++ -++ if (unlikely(id >= smap->n_buckets)) -++ return -E2BIG; -++ -++ old_bucket = xchg(&smap->buckets[id], NULL); -++ if (old_bucket) { -++ pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); -++ return 0; -++ } else { -++ return -ENOENT; -++ } -++} -++ -++/* Called when map->refcnt goes to zero, either from workqueue or from syscall */ -++static void stack_map_free(struct bpf_map *map) -++{ -++ struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); -++ -++ /* wait for bpf programs to complete before freeing stack map */ -++ synchronize_rcu(); -++ -++ bpf_map_area_free(smap->elems); -++ pcpu_freelist_destroy(&smap->freelist); -++ bpf_map_area_free(smap); -++ put_callchain_buffers(); -++} -++ -++const struct bpf_map_ops stack_trace_map_ops = { -++ .map_alloc = stack_map_alloc, -++ .map_free = stack_map_free, -++ .map_get_next_key = stack_map_get_next_key, -++ .map_lookup_elem = stack_map_lookup_elem, -++ .map_update_elem = stack_map_update_elem, -++ .map_delete_elem = stack_map_delete_elem, -++ .map_check_btf = map_check_no_btf, -++}; -++ -++static int __init stack_map_init(void) -++{ -++ int cpu; -++ struct stack_map_irq_work *work; -++ -++ for_each_possible_cpu(cpu) { -++ work = per_cpu_ptr(&up_read_work, cpu); -++ init_irq_work(&work->irq_work, do_up_read); -++ } -++ return 0; -++} -++subsys_initcall(stack_map_init); -+--- a/kernel/bpf/syscall.c -++++ b/kernel/bpf/syscall.c -+@@ -1,106 +1,333 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+- * -+- * This program is distributed in the hope that it will be useful, but -+- * WITHOUT ANY WARRANTY; without even the implied warranty of -+- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+- * General Public License for more details. -+ */ -+ #include -++#include -++#include -+ #include -+ #include -++#include -++#include -+ #include -++#include -+ #include -++#include -+ #include -+ #include -+ #include -++#include -++#include -++#include -++#include -++#include -++ -++#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \ -++ (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ -++ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ -++ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) -++#define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) -++#define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map)) -++ -++#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY) -++ -++DEFINE_PER_CPU(int, bpf_prog_active); -++static DEFINE_IDR(prog_idr); -++static DEFINE_SPINLOCK(prog_idr_lock); -++static DEFINE_IDR(map_idr); -++static DEFINE_SPINLOCK(map_idr_lock); -+ -+ int sysctl_unprivileged_bpf_disabled __read_mostly; -+ -+-static LIST_HEAD(bpf_map_types); -++static const struct bpf_map_ops * const bpf_map_types[] = { -++#define BPF_PROG_TYPE(_id, _ops) -++#define BPF_MAP_TYPE(_id, _ops) \ -++ [_id] = &_ops, -++#include -++#undef BPF_PROG_TYPE -++#undef BPF_MAP_TYPE -++}; -++ -++/* -++ * If we're handed a bigger struct than we know of, ensure all the unknown bits -++ * are 0 - i.e. new user-space does not rely on any kernel feature extensions -++ * we don't know about yet. -++ * -++ * There is a ToCToU between this function call and the following -++ * copy_from_user() call. However, this is not a concern since this function is -++ * meant to be a future-proofing of bits. -++ */ -++int bpf_check_uarg_tail_zero(void __user *uaddr, -++ size_t expected_size, -++ size_t actual_size) -++{ -++ unsigned char __user *addr; -++ unsigned char __user *end; -++ unsigned char val; -++ int err; -++ -++ if (unlikely(actual_size > PAGE_SIZE)) /* silly large */ -++ return -E2BIG; -++ -++ if (unlikely(!access_ok(VERIFY_READ, uaddr, actual_size))) -++ return -EFAULT; -++ -++ if (actual_size <= expected_size) -++ return 0; -++ -++ addr = uaddr + expected_size; -++ end = uaddr + actual_size; -++ -++ for (; addr < end; addr++) { -++ err = get_user(val, addr); -++ if (err) -++ return err; -++ if (val) -++ return -E2BIG; -++ } -++ -++ return 0; -++} -+ -+ static struct bpf_map *find_and_alloc_map(union bpf_attr *attr) -+ { -+- struct bpf_map_type_list *tl; -++ const struct bpf_map_ops *ops; -++ u32 type = attr->map_type; -+ struct bpf_map *map; -++ int err; -+ -+- list_for_each_entry(tl, &bpf_map_types, list_node) { -+- if (tl->type == attr->map_type) { -+- map = tl->ops->map_alloc(attr); -+- if (IS_ERR(map)) -+- return map; -+- map->ops = tl->ops; -+- map->map_type = attr->map_type; -+- return map; -+- } -++ if (type >= ARRAY_SIZE(bpf_map_types)) -++ return ERR_PTR(-EINVAL); -++ ops = bpf_map_types[type]; -++ if (!ops) -++ return ERR_PTR(-EINVAL); -++ -++ if (ops->map_alloc_check) { -++ err = ops->map_alloc_check(attr); -++ if (err) -++ return ERR_PTR(err); -++ } -++ map = ops->map_alloc(attr); -++ if (IS_ERR(map)) -++ return map; -++ map->ops = ops; -++ map->map_type = type; -++ return map; -++} -++ -++void *bpf_map_area_alloc(u64 size, int numa_node) -++{ -++ /* We really just want to fail instead of triggering OOM killer -++ * under memory pressure, therefore we set __GFP_NORETRY to kmalloc, -++ * which is used for lower order allocation requests. -++ * -++ * It has been observed that higher order allocation requests done by -++ * vmalloc with __GFP_NORETRY being set might fail due to not trying -++ * to reclaim memory from the page cache, thus we set -++ * __GFP_RETRY_MAYFAIL to avoid such situations. -++ */ -++ -++ const gfp_t flags = __GFP_NOWARN | __GFP_ZERO; -++ void *area; -++ -++ if (size >= SIZE_MAX) -++ return NULL; -++ -++ if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) { -++ area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags, -++ numa_node); -++ if (area != NULL) -++ return area; -+ } -+- return ERR_PTR(-EINVAL); -++ -++ return __vmalloc_node_range(size, 1, -++ VMALLOC_START, VMALLOC_END, -++ GFP_KERNEL | flags, -++ PAGE_KERNEL, 0, numa_node, -++ __builtin_return_address(0)); -+ } -+ -+-/* boot time registration of different map implementations */ -+-void bpf_register_map_type(struct bpf_map_type_list *tl) -++void bpf_map_area_free(void *area) -+ { -+- list_add(&tl->list_node, &bpf_map_types); -++ kvfree(area); -+ } -+ -+-static int bpf_map_charge_memlock(struct bpf_map *map) -++static u32 bpf_map_flags_retain_permanent(u32 flags) -+ { -+- struct user_struct *user = get_current_user(); -+- unsigned long memlock_limit; -++ /* Some map creation flags are not tied to the map object but -++ * rather to the map fd instead, so they have no meaning upon -++ * map object inspection since multiple file descriptors with -++ * different (access) properties can exist here. Thus, given -++ * this has zero meaning for the map itself, lets clear these -++ * from here. -++ */ -++ return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY); -++} -++ -++void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr) -++{ -++ map->map_type = attr->map_type; -++ map->key_size = attr->key_size; -++ map->value_size = attr->value_size; -++ map->max_entries = attr->max_entries; -++ map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags); -++ map->numa_node = bpf_map_attr_numa_node(attr); -++} -++ -++static int bpf_charge_memlock(struct user_struct *user, u32 pages) -++{ -++ unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -++ -++ if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) { -++ atomic_long_sub(pages, &user->locked_vm); -++ return -EPERM; -++ } -++ return 0; -++} -++ -++static void bpf_uncharge_memlock(struct user_struct *user, u32 pages) -++{ -++ if (user) -++ atomic_long_sub(pages, &user->locked_vm); -++} -+ -+- memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -++int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size) -++{ -++ u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT; -++ struct user_struct *user; -++ int ret; -+ -+- atomic_long_add(map->pages, &user->locked_vm); -++ if (size >= U32_MAX - PAGE_SIZE) -++ return -E2BIG; -+ -+- if (atomic_long_read(&user->locked_vm) > memlock_limit) { -+- atomic_long_sub(map->pages, &user->locked_vm); -++ user = get_current_user(); -++ ret = bpf_charge_memlock(user, pages); -++ if (ret) { -+ free_uid(user); -+- return -EPERM; -++ return ret; -+ } -+- map->user = user; -++ -++ mem->pages = pages; -++ mem->user = user; -++ -+ return 0; -+ } -+ -+-static void bpf_map_uncharge_memlock(struct bpf_map *map) -++void bpf_map_charge_finish(struct bpf_map_memory *mem) -+ { -+- struct user_struct *user = map->user; -++ bpf_uncharge_memlock(mem->user, mem->pages); -++ free_uid(mem->user); -++} -+ -+- atomic_long_sub(map->pages, &user->locked_vm); -+- free_uid(user); -++void bpf_map_charge_move(struct bpf_map_memory *dst, -++ struct bpf_map_memory *src) -++{ -++ *dst = *src; -++ -++ /* Make sure src will not be used for the redundant uncharging. */ -++ memset(src, 0, sizeof(struct bpf_map_memory)); -++} -++ -++int bpf_map_charge_memlock(struct bpf_map *map, u32 pages) -++{ -++ int ret; -++ -++ ret = bpf_charge_memlock(map->memory.user, pages); -++ if (ret) -++ return ret; -++ map->memory.pages += pages; -++ return ret; -++} -++ -++void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages) -++{ -++ bpf_uncharge_memlock(map->memory.user, pages); -++ map->memory.pages -= pages; -++} -++ -++static int bpf_map_alloc_id(struct bpf_map *map) -++{ -++ int id; -++ -++ idr_preload(GFP_KERNEL); -++ spin_lock_bh(&map_idr_lock); -++ id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC); -++ if (id > 0) -++ map->id = id; -++ spin_unlock_bh(&map_idr_lock); -++ idr_preload_end(); -++ -++ if (WARN_ON_ONCE(!id)) -++ return -ENOSPC; -++ -++ return id > 0 ? 0 : id; -++} -++ -++void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock) -++{ -++ unsigned long flags; -++ -++ /* Offloaded maps are removed from the IDR store when their device -++ * disappears - even if someone holds an fd to them they are unusable, -++ * the memory is gone, all ops will fail; they are simply waiting for -++ * refcnt to drop to be freed. -++ */ -++ if (!map->id) -++ return; -++ -++ if (do_idr_lock) -++ spin_lock_irqsave(&map_idr_lock, flags); -++ else -++ __acquire(&map_idr_lock); -++ -++ idr_remove(&map_idr, map->id); -++ map->id = 0; -++ -++ if (do_idr_lock) -++ spin_unlock_irqrestore(&map_idr_lock, flags); -++ else -++ __release(&map_idr_lock); -+ } -+ -+ /* called from workqueue */ -+ static void bpf_map_free_deferred(struct work_struct *work) -+ { -+ struct bpf_map *map = container_of(work, struct bpf_map, work); -++ struct bpf_map_memory mem; -+ -+- bpf_map_uncharge_memlock(map); -++ bpf_map_charge_move(&mem, &map->memory); -+ /* implementation dependent freeing */ -+ map->ops->map_free(map); -++ bpf_map_charge_finish(&mem); -+ } -+ -+ static void bpf_map_put_uref(struct bpf_map *map) -+ { -+ if (atomic_dec_and_test(&map->usercnt)) { -+- if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) -+- bpf_fd_array_map_clear(map); -++ if (map->ops->map_release_uref) -++ map->ops->map_release_uref(map); -+ } -+ } -+ -+ /* decrement map refcnt and schedule it for freeing via workqueue -+ * (unrelying map implementation ops->map_free() might sleep) -+ */ -+-void bpf_map_put(struct bpf_map *map) -++static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) -+ { -+ if (atomic_dec_and_test(&map->refcnt)) { -++ /* bpf_map_free_id() must be called first */ -++ bpf_map_free_id(map, do_idr_lock); -++ btf_put(map->btf); -+ INIT_WORK(&map->work, bpf_map_free_deferred); -+ schedule_work(&map->work); -+ } -+ } -+ -++void bpf_map_put(struct bpf_map *map) -++{ -++ __bpf_map_put(map, true); -++} -++EXPORT_SYMBOL_GPL(bpf_map_put); -++ -+ void bpf_map_put_with_uref(struct bpf_map *map) -+ { -+ bpf_map_put_uref(map); -+@@ -109,18 +336,110 @@ void bpf_map_put_with_uref(struct bpf_ma -+ -+ static int bpf_map_release(struct inode *inode, struct file *filp) -+ { -+- bpf_map_put_with_uref(filp->private_data); -++ struct bpf_map *map = filp->private_data; -++ -++ if (map->ops->map_release) -++ map->ops->map_release(map, filp); -++ -++ bpf_map_put_with_uref(map); -+ return 0; -+ } -+ -+-static const struct file_operations bpf_map_fops = { -+- .release = bpf_map_release, -++static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f) -++{ -++ fmode_t mode = f.file->f_mode; -++ -++ /* Our file permissions may have been overridden by global -++ * map permissions facing syscall side. -++ */ -++ if (READ_ONCE(map->frozen)) -++ mode &= ~FMODE_CAN_WRITE; -++ return mode; -++} -++ -++#ifdef CONFIG_PROC_FS -++static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp) -++{ -++ const struct bpf_map *map = filp->private_data; -++ const struct bpf_array *array; -++ u32 owner_prog_type = 0; -++ u32 owner_jited = 0; -++ -++ if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) { -++ array = container_of(map, struct bpf_array, map); -++ owner_prog_type = array->owner_prog_type; -++ owner_jited = array->owner_jited; -++ } -++ -++ seq_printf(m, -++ "map_type:\t%u\n" -++ "key_size:\t%u\n" -++ "value_size:\t%u\n" -++ "max_entries:\t%u\n" -++ "map_flags:\t%#x\n" -++ "memlock:\t%llu\n" -++ "map_id:\t%u\n" -++ "frozen:\t%u\n", -++ map->map_type, -++ map->key_size, -++ map->value_size, -++ map->max_entries, -++ map->map_flags, -++ map->memory.pages * 1ULL << PAGE_SHIFT, -++ map->id, -++ READ_ONCE(map->frozen)); -++ -++ if (owner_prog_type) { -++ seq_printf(m, "owner_prog_type:\t%u\n", -++ owner_prog_type); -++ seq_printf(m, "owner_jited:\t%u\n", -++ owner_jited); -++ } -++} -++#endif -++ -++static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz, -++ loff_t *ppos) -++{ -++ /* We need this handler such that alloc_file() enables -++ * f_mode with FMODE_CAN_READ. -++ */ -++ return -EINVAL; -++} -++ -++static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf, -++ size_t siz, loff_t *ppos) -++{ -++ /* We need this handler such that alloc_file() enables -++ * f_mode with FMODE_CAN_WRITE. -++ */ -++ return -EINVAL; -++} -++ -++const struct file_operations bpf_map_fops = { -++#ifdef CONFIG_PROC_FS -++ .show_fdinfo = bpf_map_show_fdinfo, -++#endif -++ .release = bpf_map_release, -++ .read = bpf_dummy_read, -++ .write = bpf_dummy_write, -+ }; -+ -+-int bpf_map_new_fd(struct bpf_map *map) -++int bpf_map_new_fd(struct bpf_map *map, int flags) -+ { -+ return anon_inode_getfd("bpf-map", &bpf_map_fops, map, -+- O_RDWR | O_CLOEXEC); -++ flags | O_CLOEXEC); -++} -++ -++int bpf_get_file_flag(int flags) -++{ -++ if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY)) -++ return -EINVAL; -++ if (flags & BPF_F_RDONLY) -++ return O_RDONLY; -++ if (flags & BPF_F_WRONLY) -++ return O_WRONLY; -++ return O_RDWR; -+ } -+ -+ /* helper macro to check that unused fields 'union bpf_attr' are zero */ -+@@ -131,38 +450,171 @@ int bpf_map_new_fd(struct bpf_map *map) -+ offsetof(union bpf_attr, CMD##_LAST_FIELD) - \ -+ sizeof(attr->CMD##_LAST_FIELD)) != NULL -+ -+-#define BPF_MAP_CREATE_LAST_FIELD max_entries -++/* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes. -++ * Return 0 on success and < 0 on error. -++ */ -++static int bpf_obj_name_cpy(char *dst, const char *src) -++{ -++ const char *end = src + BPF_OBJ_NAME_LEN; -++ -++ memset(dst, 0, BPF_OBJ_NAME_LEN); -++ /* Copy all isalnum(), '_' and '.' chars. */ -++ while (src < end && *src) { -++ if (!isalnum(*src) && -++ *src != '_' && *src != '.') -++ return -EINVAL; -++ *dst++ = *src++; -++ } -++ -++ /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */ -++ if (src == end) -++ return -EINVAL; -++ -++ return 0; -++} -++ -++int map_check_no_btf(const struct bpf_map *map, -++ const struct btf *btf, -++ const struct btf_type *key_type, -++ const struct btf_type *value_type) -++{ -++ return -ENOTSUPP; -++} -++ -++static int map_check_btf(struct bpf_map *map, const struct btf *btf, -++ u32 btf_key_id, u32 btf_value_id) -++{ -++ const struct btf_type *key_type, *value_type; -++ u32 key_size, value_size; -++ int ret = 0; -++ -++ /* Some maps allow key to be unspecified. */ -++ if (btf_key_id) { -++ key_type = btf_type_id_size(btf, &btf_key_id, &key_size); -++ if (!key_type || key_size != map->key_size) -++ return -EINVAL; -++ } else { -++ key_type = btf_type_by_id(btf, 0); -++ if (!map->ops->map_check_btf) -++ return -EINVAL; -++ } -++ -++ value_type = btf_type_id_size(btf, &btf_value_id, &value_size); -++ if (!value_type || value_size != map->value_size) -++ return -EINVAL; -++ -++ map->spin_lock_off = btf_find_spin_lock(btf, value_type); -++ -++ if (map_value_has_spin_lock(map)) { -++ if (map->map_flags & BPF_F_RDONLY_PROG) -++ return -EACCES; -++ if (map->map_type != BPF_MAP_TYPE_HASH && -++ map->map_type != BPF_MAP_TYPE_ARRAY && -++ map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && -++ map->map_type != BPF_MAP_TYPE_SK_STORAGE) -++ return -ENOTSUPP; -++ if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > -++ map->value_size) { -++ WARN_ONCE(1, -++ "verifier bug spin_lock_off %d value_size %d\n", -++ map->spin_lock_off, map->value_size); -++ return -EFAULT; -++ } -++ } -++ -++ if (map->ops->map_check_btf) -++ ret = map->ops->map_check_btf(map, btf, key_type, value_type); -++ -++ return ret; -++} -++ -++#define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id -+ /* called via syscall */ -+ static int map_create(union bpf_attr *attr) -+ { -++ int numa_node = bpf_map_attr_numa_node(attr); -++ struct bpf_map_memory mem; -+ struct bpf_map *map; -++ int f_flags; -+ int err; -+ -+ err = CHECK_ATTR(BPF_MAP_CREATE); -+ if (err) -+ return -EINVAL; -+ -++ f_flags = bpf_get_file_flag(attr->map_flags); -++ if (f_flags < 0) -++ return f_flags; -++ -++ if (numa_node != NUMA_NO_NODE && -++ ((unsigned int)numa_node >= nr_node_ids || -++ !node_online(numa_node))) -++ return -EINVAL; -++ -+ /* find map type and init map: hashtable vs rbtree vs bloom vs ... */ -+ map = find_and_alloc_map(attr); -+ if (IS_ERR(map)) -+ return PTR_ERR(map); -+ -++ err = bpf_obj_name_cpy(map->name, attr->map_name); -++ if (err) -++ goto free_map; -++ -+ atomic_set(&map->refcnt, 1); -+ atomic_set(&map->usercnt, 1); -+ -+- err = bpf_map_charge_memlock(map); -++ if (attr->btf_key_type_id || attr->btf_value_type_id) { -++ struct btf *btf; -++ -++ if (!attr->btf_value_type_id) { -++ err = -EINVAL; -++ goto free_map; -++ } -++ -++ btf = btf_get_by_fd(attr->btf_fd); -++ if (IS_ERR(btf)) { -++ err = PTR_ERR(btf); -++ goto free_map; -++ } -++ -++ err = map_check_btf(map, btf, attr->btf_key_type_id, -++ attr->btf_value_type_id); -++ if (err) { -++ btf_put(btf); -++ goto free_map; -++ } -++ -++ map->btf = btf; -++ map->btf_key_type_id = attr->btf_key_type_id; -++ map->btf_value_type_id = attr->btf_value_type_id; -++ } else { -++ map->spin_lock_off = -EINVAL; -++ } -++ -++ err = bpf_map_alloc_id(map); -+ if (err) -+- goto free_map; -++ goto free_map_sec; -+ -+- err = bpf_map_new_fd(map); -+- if (err < 0) -+- /* failed to allocate fd */ -+- goto free_map; -++ err = bpf_map_new_fd(map, f_flags); -++ if (err < 0) { -++ /* failed to allocate fd. -++ * bpf_map_put_with_uref() is needed because the above -++ * bpf_map_alloc_id() has published the map -++ * to the userspace and the userspace may -++ * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID. -++ */ -++ bpf_map_put_with_uref(map); -++ return err; -++ } -+ -+ return err; -+ -++free_map_sec: -+ free_map: -++ btf_put(map->btf); -++ bpf_map_charge_move(&mem, &map->memory); -+ map->ops->map_free(map); -++ bpf_map_charge_finish(&mem); -+ return err; -+ } -+ -+@@ -194,6 +646,7 @@ struct bpf_map *bpf_map_inc(struct bpf_m -+ atomic_inc(&map->usercnt); -+ return map; -+ } -++EXPORT_SYMBOL_GPL(bpf_map_inc); -+ -+ struct bpf_map *bpf_map_get_with_uref(u32 ufd) -+ { -+@@ -210,59 +663,155 @@ struct bpf_map *bpf_map_get_with_uref(u3 -+ return map; -+ } -+ -+-/* helper to convert user pointers passed inside __aligned_u64 fields */ -+-static void __user *u64_to_ptr(__u64 val) -++/* map_idr_lock should have been held */ -++static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, -++ bool uref) -++{ -++ int refold; -++ -++ refold = atomic_fetch_add_unless(&map->refcnt, 1, 0); -++ -++ if (refold >= BPF_MAX_REFCNT) { -++ __bpf_map_put(map, false); -++ return ERR_PTR(-EBUSY); -++ } -++ -++ if (!refold) -++ return ERR_PTR(-ENOENT); -++ -++ if (uref) -++ atomic_inc(&map->usercnt); -++ -++ return map; -++} -++ -++struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map, bool uref) -++{ -++ spin_lock_bh(&map_idr_lock); -++ map = __bpf_map_inc_not_zero(map, uref); -++ spin_unlock_bh(&map_idr_lock); -++ -++ return map; -++} -++EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero); -++ -++int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value) -++{ -++ return -ENOTSUPP; -++} -++ -++static void *__bpf_copy_key(void __user *ukey, u64 key_size) -+ { -+- return (void __user *) (unsigned long) val; -++ if (key_size) -++ return memdup_user(ukey, key_size); -++ -++ if (ukey) -++ return ERR_PTR(-EINVAL); -++ -++ return NULL; -+ } -+ -+ /* last field in 'union bpf_attr' used by this command */ -+-#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value -++#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags -+ -+ static int map_lookup_elem(union bpf_attr *attr) -+ { -+- void __user *ukey = u64_to_ptr(attr->key); -+- void __user *uvalue = u64_to_ptr(attr->value); -++ void __user *ukey = u64_to_user_ptr(attr->key); -++ void __user *uvalue = u64_to_user_ptr(attr->value); -+ int ufd = attr->map_fd; -+ struct bpf_map *map; -+ void *key, *value, *ptr; -++ u32 value_size; -+ struct fd f; -+ int err; -+ -+ if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM)) -+ return -EINVAL; -+ -++ if (attr->flags & ~BPF_F_LOCK) -++ return -EINVAL; -++ -+ f = fdget(ufd); -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) -+ return PTR_ERR(map); -++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { -++ err = -EPERM; -++ goto err_put; -++ } -+ -+- err = -ENOMEM; -+- key = kmalloc(map->key_size, GFP_USER); -+- if (!key) -++ if ((attr->flags & BPF_F_LOCK) && -++ !map_value_has_spin_lock(map)) { -++ err = -EINVAL; -+ goto err_put; -++ } -+ -+- err = -EFAULT; -+- if (copy_from_user(key, ukey, map->key_size) != 0) -+- goto free_key; -++ key = __bpf_copy_key(ukey, map->key_size); -++ if (IS_ERR(key)) { -++ err = PTR_ERR(key); -++ goto err_put; -++ } -++ -++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) -++ value_size = round_up(map->value_size, 8) * num_possible_cpus(); -++ else if (IS_FD_MAP(map)) -++ value_size = sizeof(u32); -++ else -++ value_size = map->value_size; -+ -+ err = -ENOMEM; -+- value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN); -++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); -+ if (!value) -+ goto free_key; -+ -+- rcu_read_lock(); -+- ptr = map->ops->map_lookup_elem(map, key); -+- if (ptr) -+- memcpy(value, ptr, map->value_size); -+- rcu_read_unlock(); -++ preempt_disable(); -++ this_cpu_inc(bpf_prog_active); -++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { -++ err = bpf_percpu_hash_copy(map, key, value); -++ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { -++ err = bpf_percpu_array_copy(map, key, value); -++ } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) { -++ err = bpf_stackmap_copy(map, key, value); -++ } else if (IS_FD_ARRAY(map)) { -++ err = bpf_fd_array_map_lookup_elem(map, key, value); -++ } else if (IS_FD_HASH(map)) { -++ err = bpf_fd_htab_map_lookup_elem(map, key, value); -++ } else if (map->map_type == BPF_MAP_TYPE_QUEUE || -++ map->map_type == BPF_MAP_TYPE_STACK) { -++ err = map->ops->map_peek_elem(map, value); -++ } else { -++ rcu_read_lock(); -++ if (map->ops->map_lookup_elem_sys_only) -++ ptr = map->ops->map_lookup_elem_sys_only(map, key); -++ else -++ ptr = map->ops->map_lookup_elem(map, key); -++ if (IS_ERR(ptr)) { -++ err = PTR_ERR(ptr); -++ } else if (!ptr) { -++ err = -ENOENT; -++ } else { -++ err = 0; -++ if (attr->flags & BPF_F_LOCK) -++ /* lock 'ptr' and copy everything but lock */ -++ copy_map_value_locked(map, value, ptr, true); -++ else -++ copy_map_value(map, value, ptr); -++ /* mask lock, since value wasn't zero inited */ -++ check_and_init_map_lock(map, value); -++ } -++ rcu_read_unlock(); -++ } -++ this_cpu_dec(bpf_prog_active); -++ preempt_enable(); -+ -+- err = -ENOENT; -+- if (!ptr) -++ if (err) -+ goto free_value; -+ -+ err = -EFAULT; -+- if (copy_to_user(uvalue, value, map->value_size) != 0) -++ if (copy_to_user(uvalue, value, value_size) != 0) -+ goto free_value; -+ -+ err = 0; -+@@ -276,15 +825,27 @@ err_put: -+ return err; -+ } -+ -++static void maybe_wait_bpf_programs(struct bpf_map *map) -++{ -++ /* Wait for any running BPF programs to complete so that -++ * userspace, when we return to it, knows that all programs -++ * that could be running use the new map value. -++ */ -++ if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS || -++ map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) -++ synchronize_rcu(); -++} -++ -+ #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags -+ -+ static int map_update_elem(union bpf_attr *attr) -+ { -+- void __user *ukey = u64_to_ptr(attr->key); -+- void __user *uvalue = u64_to_ptr(attr->value); -++ void __user *ukey = u64_to_user_ptr(attr->key); -++ void __user *uvalue = u64_to_user_ptr(attr->value); -+ int ufd = attr->map_fd; -+ struct bpf_map *map; -+ void *key, *value; -++ u32 value_size; -+ struct fd f; -+ int err; -+ -+@@ -295,32 +856,79 @@ static int map_update_elem(union bpf_att -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) -+ return PTR_ERR(map); -++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { -++ err = -EPERM; -++ goto err_put; -++ } -+ -+- err = -ENOMEM; -+- key = kmalloc(map->key_size, GFP_USER); -+- if (!key) -++ if ((attr->flags & BPF_F_LOCK) && -++ !map_value_has_spin_lock(map)) { -++ err = -EINVAL; -++ goto err_put; -++ } -++ -++ key = __bpf_copy_key(ukey, map->key_size); -++ if (IS_ERR(key)) { -++ err = PTR_ERR(key); -+ goto err_put; -++ } -+ -+- err = -EFAULT; -+- if (copy_from_user(key, ukey, map->key_size) != 0) -+- goto free_key; -++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) -++ value_size = round_up(map->value_size, 8) * num_possible_cpus(); -++ else -++ value_size = map->value_size; -+ -+ err = -ENOMEM; -+- value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN); -++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); -+ if (!value) -+ goto free_key; -+ -+ err = -EFAULT; -+- if (copy_from_user(value, uvalue, map->value_size) != 0) -++ if (copy_from_user(value, uvalue, value_size) != 0) -+ goto free_value; -+ -+- /* eBPF program that use maps are running under rcu_read_lock(), -+- * therefore all map accessors rely on this fact, so do the same here -+- */ -+- rcu_read_lock(); -+- err = map->ops->map_update_elem(map, key, value, attr->flags); -+- rcu_read_unlock(); -++ /* Need to create a kthread, thus must support schedule */ -++ if (map->map_type == BPF_MAP_TYPE_CPUMAP || -++ map->map_type == BPF_MAP_TYPE_SOCKHASH || -++ map->map_type == BPF_MAP_TYPE_SOCKMAP) { -++ err = map->ops->map_update_elem(map, key, value, attr->flags); -++ goto out; -++ } -+ -++ /* must increment bpf_prog_active to avoid kprobe+bpf triggering from -++ * inside bpf map update or delete otherwise deadlocks are possible -++ */ -++ preempt_disable(); -++ __this_cpu_inc(bpf_prog_active); -++ if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || -++ map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { -++ err = bpf_percpu_hash_update(map, key, value, attr->flags); -++ } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { -++ err = bpf_percpu_array_update(map, key, value, attr->flags); -++ } else if (IS_FD_ARRAY(map)) { -++ rcu_read_lock(); -++ err = bpf_fd_array_map_update_elem(map, f.file, key, value, -++ attr->flags); -++ rcu_read_unlock(); -++ } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { -++ rcu_read_lock(); -++ err = bpf_fd_htab_map_update_elem(map, f.file, key, value, -++ attr->flags); -++ rcu_read_unlock(); -++ } else if (map->map_type == BPF_MAP_TYPE_QUEUE || -++ map->map_type == BPF_MAP_TYPE_STACK) { -++ err = map->ops->map_push_elem(map, value, attr->flags); -++ } else { -++ rcu_read_lock(); -++ err = map->ops->map_update_elem(map, key, value, attr->flags); -++ rcu_read_unlock(); -++ } -++ __this_cpu_dec(bpf_prog_active); -++ preempt_enable(); -++ maybe_wait_bpf_programs(map); -++out: -+ free_value: -+ kfree(value); -+ free_key: -+@@ -334,7 +942,7 @@ err_put: -+ -+ static int map_delete_elem(union bpf_attr *attr) -+ { -+- void __user *ukey = u64_to_ptr(attr->key); -++ void __user *ukey = u64_to_user_ptr(attr->key); -+ int ufd = attr->map_fd; -+ struct bpf_map *map; -+ struct fd f; -+@@ -348,21 +956,25 @@ static int map_delete_elem(union bpf_att -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) -+ return PTR_ERR(map); -+- -+- err = -ENOMEM; -+- key = kmalloc(map->key_size, GFP_USER); -+- if (!key) -++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { -++ err = -EPERM; -+ goto err_put; -++ } -+ -+- err = -EFAULT; -+- if (copy_from_user(key, ukey, map->key_size) != 0) -+- goto free_key; -++ key = __bpf_copy_key(ukey, map->key_size); -++ if (IS_ERR(key)) { -++ err = PTR_ERR(key); -++ goto err_put; -++ } -+ -++ preempt_disable(); -++ __this_cpu_inc(bpf_prog_active); -+ rcu_read_lock(); -+ err = map->ops->map_delete_elem(map, key); -+ rcu_read_unlock(); -+- -+-free_key: -++ __this_cpu_dec(bpf_prog_active); -++ preempt_enable(); -++ maybe_wait_bpf_programs(map); -+ kfree(key); -+ err_put: -+ fdput(f); -+@@ -374,8 +986,8 @@ err_put: -+ -+ static int map_get_next_key(union bpf_attr *attr) -+ { -+- void __user *ukey = u64_to_ptr(attr->key); -+- void __user *unext_key = u64_to_ptr(attr->next_key); -++ void __user *ukey = u64_to_user_ptr(attr->key); -++ void __user *unext_key = u64_to_user_ptr(attr->next_key); -+ int ufd = attr->map_fd; -+ struct bpf_map *map; -+ void *key, *next_key; -+@@ -389,15 +1001,20 @@ static int map_get_next_key(union bpf_at -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) -+ return PTR_ERR(map); -+- -+- err = -ENOMEM; -+- key = kmalloc(map->key_size, GFP_USER); -+- if (!key) -++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) { -++ err = -EPERM; -+ goto err_put; -++ } -+ -+- err = -EFAULT; -+- if (copy_from_user(key, ukey, map->key_size) != 0) -+- goto free_key; -++ if (ukey) { -++ key = __bpf_copy_key(ukey, map->key_size); -++ if (IS_ERR(key)) { -++ err = PTR_ERR(key); -++ goto err_put; -++ } -++ } else { -++ key = NULL; -++ } -+ -+ err = -ENOMEM; -+ next_key = kmalloc(map->key_size, GFP_USER); -+@@ -425,77 +1042,126 @@ err_put: -+ return err; -+ } -+ -+-static LIST_HEAD(bpf_prog_types); -++#define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value -+ -+-static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) -++static int map_lookup_and_delete_elem(union bpf_attr *attr) -+ { -+- struct bpf_prog_type_list *tl; -++ void __user *ukey = u64_to_user_ptr(attr->key); -++ void __user *uvalue = u64_to_user_ptr(attr->value); -++ int ufd = attr->map_fd; -++ struct bpf_map *map; -++ void *key, *value; -++ u32 value_size; -++ struct fd f; -++ int err; -+ -+- list_for_each_entry(tl, &bpf_prog_types, list_node) { -+- if (tl->type == type) { -+- prog->aux->ops = tl->ops; -+- prog->type = type; -+- return 0; -+- } -++ if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM)) -++ return -EINVAL; -++ -++ f = fdget(ufd); -++ map = __bpf_map_get(f); -++ if (IS_ERR(map)) -++ return PTR_ERR(map); -++ if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ) || -++ !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) { -++ err = -EPERM; -++ goto err_put; -+ } -+ -+- return -EINVAL; -+-} -++ key = __bpf_copy_key(ukey, map->key_size); -++ if (IS_ERR(key)) { -++ err = PTR_ERR(key); -++ goto err_put; -++ } -+ -+-void bpf_register_prog_type(struct bpf_prog_type_list *tl) -+-{ -+- list_add(&tl->list_node, &bpf_prog_types); -++ value_size = map->value_size; -++ -++ err = -ENOMEM; -++ value = kmalloc(value_size, GFP_USER | __GFP_NOWARN); -++ if (!value) -++ goto free_key; -++ -++ if (map->map_type == BPF_MAP_TYPE_QUEUE || -++ map->map_type == BPF_MAP_TYPE_STACK) { -++ err = map->ops->map_pop_elem(map, value); -++ } else { -++ err = -ENOTSUPP; -++ } -++ -++ if (err) -++ goto free_value; -++ -++ if (copy_to_user(uvalue, value, value_size) != 0) { -++ err = -EFAULT; -++ goto free_value; -++ } -++ -++ err = 0; -++ -++free_value: -++ kfree(value); -++free_key: -++ kfree(key); -++err_put: -++ fdput(f); -++ return err; -+ } -+ -+-/* fixup insn->imm field of bpf_call instructions: -+- * if (insn->imm == BPF_FUNC_map_lookup_elem) -+- * insn->imm = bpf_map_lookup_elem - __bpf_call_base; -+- * else if (insn->imm == BPF_FUNC_map_update_elem) -+- * insn->imm = bpf_map_update_elem - __bpf_call_base; -+- * else ... -+- * -+- * this function is called after eBPF program passed verification -+- */ -+-static void fixup_bpf_calls(struct bpf_prog *prog) -++#define BPF_MAP_FREEZE_LAST_FIELD map_fd -++ -++static int map_freeze(const union bpf_attr *attr) -+ { -+- const struct bpf_func_proto *fn; -+- int i; -++ int err = 0, ufd = attr->map_fd; -++ struct bpf_map *map; -++ struct fd f; -+ -+- for (i = 0; i < prog->len; i++) { -+- struct bpf_insn *insn = &prog->insnsi[i]; -++ if (CHECK_ATTR(BPF_MAP_FREEZE)) -++ return -EINVAL; -+ -+- if (insn->code == (BPF_JMP | BPF_CALL)) { -+- /* we reach here when program has bpf_call instructions -+- * and it passed bpf_check(), means that -+- * ops->get_func_proto must have been supplied, check it -+- */ -+- BUG_ON(!prog->aux->ops->get_func_proto); -++ f = fdget(ufd); -++ map = __bpf_map_get(f); -++ if (IS_ERR(map)) -++ return PTR_ERR(map); -++ if (READ_ONCE(map->frozen)) { -++ err = -EBUSY; -++ goto err_put; -++ } -++ if (!capable(CAP_SYS_ADMIN)) { -++ err = -EPERM; -++ goto err_put; -++ } -+ -+- if (insn->imm == BPF_FUNC_get_route_realm) -+- prog->dst_needed = 1; -+- if (insn->imm == BPF_FUNC_get_prandom_u32) -+- bpf_user_rnd_init_once(); -+- if (insn->imm == BPF_FUNC_tail_call) { -+- /* mark bpf_tail_call as different opcode -+- * to avoid conditional branch in -+- * interpeter for every normal call -+- * and to prevent accidental JITing by -+- * JIT compiler that doesn't support -+- * bpf_tail_call yet -+- */ -+- insn->imm = 0; -+- insn->code |= BPF_X; -+- continue; -+- } -++ WRITE_ONCE(map->frozen, true); -++err_put: -++ fdput(f); -++ return err; -++} -+ -+- fn = prog->aux->ops->get_func_proto(insn->imm); -+- /* all functions that have prototype and verifier allowed -+- * programs to call them, must be real in-kernel functions -+- */ -+- BUG_ON(!fn->func); -+- insn->imm = fn->func - __bpf_call_base; -+- } -+- } -++static const struct bpf_prog_ops * const bpf_prog_types[] = { -++#define BPF_PROG_TYPE(_id, _name) \ -++ [_id] = & _name ## _prog_ops, -++#define BPF_MAP_TYPE(_id, _ops) -++#include -++#undef BPF_PROG_TYPE -++#undef BPF_MAP_TYPE -++}; -++ -++static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) -++{ -++ const struct bpf_prog_ops *ops; -++ -++ if (type >= ARRAY_SIZE(bpf_prog_types)) -++ return -EINVAL; -++ ops = bpf_prog_types[type]; -++ if (!ops) -++ return -EINVAL; -++ -++ if (!bpf_prog_is_dev_bound(prog->aux)) -++ prog->aux->ops = ops; -++ else -++ return -EINVAL; -++ prog->type = type; -++ return 0; -+ } -+ -+ /* drop refcnt on maps used by eBPF program and free auxilary data */ -+@@ -509,19 +1175,39 @@ static void free_used_maps(struct bpf_pr -+ kfree(aux->used_maps); -+ } -+ -++int __bpf_prog_charge(struct user_struct *user, u32 pages) -++{ -++ unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -++ unsigned long user_bufs; -++ -++ if (user) { -++ user_bufs = atomic_long_add_return(pages, &user->locked_vm); -++ if (user_bufs > memlock_limit) { -++ atomic_long_sub(pages, &user->locked_vm); -++ return -EPERM; -++ } -++ } -++ -++ return 0; -++} -++ -++void __bpf_prog_uncharge(struct user_struct *user, u32 pages) -++{ -++ if (user) -++ atomic_long_sub(pages, &user->locked_vm); -++} -++ -+ static int bpf_prog_charge_memlock(struct bpf_prog *prog) -+ { -+ struct user_struct *user = get_current_user(); -+- unsigned long memlock_limit; -++ int ret; -+ -+- memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; -+- -+- atomic_long_add(prog->pages, &user->locked_vm); -+- if (atomic_long_read(&user->locked_vm) > memlock_limit) { -+- atomic_long_sub(prog->pages, &user->locked_vm); -++ ret = __bpf_prog_charge(user, prog->pages); -++ if (ret) { -+ free_uid(user); -+- return -EPERM; -++ return ret; -+ } -++ -+ prog->aux->user = user; -+ return 0; -+ } -+@@ -530,30 +1216,87 @@ static void bpf_prog_uncharge_memlock(st -+ { -+ struct user_struct *user = prog->aux->user; -+ -+- atomic_long_sub(prog->pages, &user->locked_vm); -++ __bpf_prog_uncharge(user, prog->pages); -+ free_uid(user); -+ } -+ -+-static void __prog_put_common(struct rcu_head *rcu) -++static int bpf_prog_alloc_id(struct bpf_prog *prog) -++{ -++ int id; -++ -++ idr_preload(GFP_KERNEL); -++ spin_lock_bh(&prog_idr_lock); -++ id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC); -++ if (id > 0) -++ prog->aux->id = id; -++ spin_unlock_bh(&prog_idr_lock); -++ idr_preload_end(); -++ -++ /* id is in [1, INT_MAX) */ -++ if (WARN_ON_ONCE(!id)) -++ return -ENOSPC; -++ -++ return id > 0 ? 0 : id; -++} -++ -++void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock) -++{ -++ /* cBPF to eBPF migrations are currently not in the idr store. -++ * Offloaded programs are removed from the store when their device -++ * disappears - even if someone grabs an fd to them they are unusable, -++ * simply waiting for refcnt to drop to be freed. -++ */ -++ if (!prog->aux->id) -++ return; -++ -++ if (do_idr_lock) -++ spin_lock_bh(&prog_idr_lock); -++ else -++ __acquire(&prog_idr_lock); -++ -++ idr_remove(&prog_idr, prog->aux->id); -++ prog->aux->id = 0; -++ -++ if (do_idr_lock) -++ spin_unlock_bh(&prog_idr_lock); -++ else -++ __release(&prog_idr_lock); -++} -++ -++static void __bpf_prog_put_rcu(struct rcu_head *rcu) -+ { -+ struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); -+ -++ kvfree(aux->func_info); -+ free_used_maps(aux); -+ bpf_prog_uncharge_memlock(aux->prog); -+ bpf_prog_free(aux->prog); -+ } -+ -+-/* version of bpf_prog_put() that is called after a grace period */ -+-void bpf_prog_put_rcu(struct bpf_prog *prog) -++static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) -++{ -++ bpf_prog_kallsyms_del_all(prog); -++ btf_put(prog->aux->btf); -++ bpf_prog_free_linfo(prog); -++ -++ if (deferred) -++ call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); -++ else -++ __bpf_prog_put_rcu(&prog->aux->rcu); -++} -++ -++static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) -+ { -+- if (atomic_dec_and_test(&prog->aux->refcnt)) -+- call_rcu(&prog->aux->rcu, __prog_put_common); -++ if (atomic_dec_and_test(&prog->aux->refcnt)) { -++ /* bpf_prog_free_id() must be called first */ -++ bpf_prog_free_id(prog, do_idr_lock); -++ __bpf_prog_put_noref(prog, true); -++ } -+ } -+ -+ void bpf_prog_put(struct bpf_prog *prog) -+ { -+- if (atomic_dec_and_test(&prog->aux->refcnt)) -+- __prog_put_common(&prog->aux->rcu); -++ __bpf_prog_put(prog, true); -+ } -+ EXPORT_SYMBOL_GPL(bpf_prog_put); -+ -+@@ -561,12 +1304,68 @@ static int bpf_prog_release(struct inode -+ { -+ struct bpf_prog *prog = filp->private_data; -+ -+- bpf_prog_put_rcu(prog); -++ bpf_prog_put(prog); -+ return 0; -+ } -+ -+-static const struct file_operations bpf_prog_fops = { -+- .release = bpf_prog_release, -++static void bpf_prog_get_stats(const struct bpf_prog *prog, -++ struct bpf_prog_stats *stats) -++{ -++ u64 nsecs = 0, cnt = 0; -++ int cpu; -++ -++ for_each_possible_cpu(cpu) { -++ const struct bpf_prog_stats *st; -++ unsigned int start; -++ u64 tnsecs, tcnt; -++ -++ st = per_cpu_ptr(prog->aux->stats, cpu); -++ do { -++ start = u64_stats_fetch_begin_irq(&st->syncp); -++ tnsecs = st->nsecs; -++ tcnt = st->cnt; -++ } while (u64_stats_fetch_retry_irq(&st->syncp, start)); -++ nsecs += tnsecs; -++ cnt += tcnt; -++ } -++ stats->nsecs = nsecs; -++ stats->cnt = cnt; -++} -++ -++#ifdef CONFIG_PROC_FS -++static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) -++{ -++ const struct bpf_prog *prog = filp->private_data; -++ char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; -++ struct bpf_prog_stats stats; -++ -++ bpf_prog_get_stats(prog, &stats); -++ bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); -++ seq_printf(m, -++ "prog_type:\t%u\n" -++ "prog_jited:\t%u\n" -++ "prog_tag:\t%s\n" -++ "memlock:\t%llu\n" -++ "prog_id:\t%u\n" -++ "run_time_ns:\t%llu\n" -++ "run_cnt:\t%llu\n", -++ prog->type, -++ prog->jited, -++ prog_tag, -++ prog->pages * 1ULL << PAGE_SHIFT, -++ prog->aux->id, -++ stats.nsecs, -++ stats.cnt); -++} -++#endif -++ -++const struct file_operations bpf_prog_fops = { -++#ifdef CONFIG_PROC_FS -++ .show_fdinfo = bpf_prog_show_fdinfo, -++#endif -++ .release = bpf_prog_release, -++ .read = bpf_dummy_read, -++ .write = bpf_dummy_write, -+ }; -+ -+ int bpf_prog_new_fd(struct bpf_prog *prog) -+@@ -575,7 +1374,7 @@ int bpf_prog_new_fd(struct bpf_prog *pro -+ O_RDWR | O_CLOEXEC); -+ } -+ -+-static struct bpf_prog *__bpf_prog_get(struct fd f) -++static struct bpf_prog *____bpf_prog_get(struct fd f) -+ { -+ if (!f.file) -+ return ERR_PTR(-EBADF); -+@@ -587,38 +1386,178 @@ static struct bpf_prog *__bpf_prog_get(s -+ return f.file->private_data; -+ } -+ -++struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i) -++{ -++ if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) { -++ atomic_sub(i, &prog->aux->refcnt); -++ return ERR_PTR(-EBUSY); -++ } -++ return prog; -++} -++EXPORT_SYMBOL_GPL(bpf_prog_add); -++ -++void bpf_prog_sub(struct bpf_prog *prog, int i) -++{ -++ /* Only to be used for undoing previous bpf_prog_add() in some -++ * error path. We still know that another entity in our call -++ * path holds a reference to the program, thus atomic_sub() can -++ * be safely used in such cases! -++ */ -++ WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0); -++} -++EXPORT_SYMBOL_GPL(bpf_prog_sub); -++ -+ struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog) -+ { -+- if (atomic_inc_return(&prog->aux->refcnt) > BPF_MAX_REFCNT) { -+- atomic_dec(&prog->aux->refcnt); -++ return bpf_prog_add(prog, 1); -++} -++EXPORT_SYMBOL_GPL(bpf_prog_inc); -++ -++/* prog_idr_lock should have been held */ -++struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog) -++{ -++ int refold; -++ -++ refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0); -++ -++ if (refold >= BPF_MAX_REFCNT) { -++ __bpf_prog_put(prog, false); -+ return ERR_PTR(-EBUSY); -+ } -++ -++ if (!refold) -++ return ERR_PTR(-ENOENT); -++ -+ return prog; -+ } -++EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero); -+ -+-/* called by sockets/tracing/seccomp before attaching program to an event -+- * pairs with bpf_prog_put() -+- */ -+-struct bpf_prog *bpf_prog_get(u32 ufd) -++bool bpf_prog_get_ok(struct bpf_prog *prog, -++ enum bpf_prog_type *attach_type, bool attach_drv) -++{ -++ /* not an attachment, just a refcount inc, always allow */ -++ if (!attach_type) -++ return true; -++ -++ if (prog->type != *attach_type) -++ return false; -++ if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv) -++ return false; -++ -++ return true; -++} -++ -++static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type, -++ bool attach_drv) -+ { -+ struct fd f = fdget(ufd); -+ struct bpf_prog *prog; -+ -+- prog = __bpf_prog_get(f); -++ prog = ____bpf_prog_get(f); -+ if (IS_ERR(prog)) -+ return prog; -++ if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) { -++ prog = ERR_PTR(-EINVAL); -++ goto out; -++ } -+ -+ prog = bpf_prog_inc(prog); -++out: -+ fdput(f); -+- -+ return prog; -+ } -+-EXPORT_SYMBOL_GPL(bpf_prog_get); -++ -++struct bpf_prog *bpf_prog_get(u32 ufd) -++{ -++ return __bpf_prog_get(ufd, NULL, false); -++} -++ -++struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, -++ bool attach_drv) -++{ -++ return __bpf_prog_get(ufd, &type, attach_drv); -++} -++EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev); -++ -++/* Initially all BPF programs could be loaded w/o specifying -++ * expected_attach_type. Later for some of them specifying expected_attach_type -++ * at load time became required so that program could be validated properly. -++ * Programs of types that are allowed to be loaded both w/ and w/o (for -++ * backward compatibility) expected_attach_type, should have the default attach -++ * type assigned to expected_attach_type for the latter case, so that it can be -++ * validated later at attach time. -++ * -++ * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if -++ * prog type requires it but has some attach types that have to be backward -++ * compatible. -++ */ -++static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) -++{ -++ switch (attr->prog_type) { -++ case BPF_PROG_TYPE_CGROUP_SOCK: -++ /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't -++ * exist so checking for non-zero is the way to go here. -++ */ -++ if (!attr->expected_attach_type) -++ attr->expected_attach_type = -++ BPF_CGROUP_INET_SOCK_CREATE; -++ break; -++ } -++} -++ -++static int -++bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, -++ enum bpf_attach_type expected_attach_type) -++{ -++ switch (prog_type) { -++ case BPF_PROG_TYPE_CGROUP_SOCK: -++ switch (expected_attach_type) { -++ case BPF_CGROUP_INET_SOCK_CREATE: -++ case BPF_CGROUP_INET4_POST_BIND: -++ case BPF_CGROUP_INET6_POST_BIND: -++ return 0; -++ default: -++ return -EINVAL; -++ } -++ case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: -++ switch (expected_attach_type) { -++ case BPF_CGROUP_INET4_BIND: -++ case BPF_CGROUP_INET6_BIND: -++ case BPF_CGROUP_INET4_CONNECT: -++ case BPF_CGROUP_INET6_CONNECT: -++ case BPF_CGROUP_UDP4_SENDMSG: -++ case BPF_CGROUP_UDP6_SENDMSG: -++ case BPF_CGROUP_UDP4_RECVMSG: -++ case BPF_CGROUP_UDP6_RECVMSG: -++ return 0; -++ default: -++ return -EINVAL; -++ } -++ case BPF_PROG_TYPE_CGROUP_SKB: -++ switch (expected_attach_type) { -++ case BPF_CGROUP_INET_INGRESS: -++ case BPF_CGROUP_INET_EGRESS: -++ return 0; -++ default: -++ return -EINVAL; -++ } -++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: -++ switch (expected_attach_type) { -++ case BPF_CGROUP_SETSOCKOPT: -++ case BPF_CGROUP_GETSOCKOPT: -++ return 0; -++ default: -++ return -EINVAL; -++ } -++ default: -++ return 0; -++ } -++} -+ -+ /* last field in 'union bpf_attr' used by this command */ -+-#define BPF_PROG_LOAD_LAST_FIELD kern_version -++#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt -+ -+-static int bpf_prog_load(union bpf_attr *attr) -++static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) -+ { -+ enum bpf_prog_type type = attr->prog_type; -+ struct bpf_prog *prog; -+@@ -629,8 +1568,19 @@ static int bpf_prog_load(union bpf_attr -+ if (CHECK_ATTR(BPF_PROG_LOAD)) -+ return -EINVAL; -+ -++ if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | -++ BPF_F_ANY_ALIGNMENT | -++ BPF_F_TEST_STATE_FREQ | -++ BPF_F_TEST_RND_HI32)) -++ return -EINVAL; -++ -++ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && -++ (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && -++ !capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -+ /* copy eBPF program license from user space */ -+- if (strncpy_from_user(license, u64_to_ptr(attr->license), -++ if (strncpy_from_user(license, u64_to_user_ptr(attr->license), -+ sizeof(license) - 1) < 0) -+ return -EFAULT; -+ license[sizeof(license) - 1] = 0; -+@@ -638,30 +1588,36 @@ static int bpf_prog_load(union bpf_attr -+ /* eBPF programs must be GPL compatible to use GPL-ed functions */ -+ is_gpl = license_is_gpl_compatible(license); -+ -+- if (attr->insn_cnt >= BPF_MAXINSNS) -+- return -EINVAL; -++ if (attr->insn_cnt == 0 || -++ attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) -++ return -E2BIG; -++ if (type != BPF_PROG_TYPE_SOCKET_FILTER && -++ type != BPF_PROG_TYPE_CGROUP_SKB && -++ !capable(CAP_SYS_ADMIN)) -++ return -EPERM; -+ -+- if (type == BPF_PROG_TYPE_KPROBE && -+- attr->kern_version != LINUX_VERSION_CODE) -++ bpf_prog_load_fixup_attach_type(attr); -++ if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type)) -+ return -EINVAL; -+ -+- if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN)) -+- return -EPERM; -+- -+ /* plain bpf_prog allocation */ -+ prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); -+ if (!prog) -+ return -ENOMEM; -+ -++ prog->expected_attach_type = attr->expected_attach_type; -++ -++ prog->aux->offload_requested = !!attr->prog_ifindex; -++ -+ err = bpf_prog_charge_memlock(prog); -+ if (err) -+- goto free_prog_nouncharge; -++ goto free_prog_sec; -+ -+ prog->len = attr->insn_cnt; -+ -+ err = -EFAULT; -+- if (copy_from_user(prog->insns, u64_to_ptr(attr->insns), -+- prog->len * sizeof(struct bpf_insn)) != 0) -++ if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns), -++ bpf_prog_insn_size(prog)) != 0) -+ goto free_prog; -+ -+ prog->orig_prog = NULL; -+@@ -675,91 +1631,720 @@ static int bpf_prog_load(union bpf_attr -+ if (err < 0) -+ goto free_prog; -+ -++ prog->aux->load_time = ktime_get_boot_ns(); -++ err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name); -++ if (err) -++ goto free_prog; -++ -+ /* run eBPF verifier */ -+- err = bpf_check(&prog, attr); -++ err = bpf_check(&prog, attr, uattr); -+ if (err < 0) -+ goto free_used_maps; -+ -+- /* fixup BPF_CALL->imm field */ -+- fixup_bpf_calls(prog); -+- -+- /* eBPF program is ready to be JITed */ -+- err = bpf_prog_select_runtime(prog); -++ prog = bpf_prog_select_runtime(prog, &err); -+ if (err < 0) -+ goto free_used_maps; -+ -+- err = bpf_prog_new_fd(prog); -+- if (err < 0) -+- /* failed to allocate fd */ -++ err = bpf_prog_alloc_id(prog); -++ if (err) -+ goto free_used_maps; -+ -++ /* Upon success of bpf_prog_alloc_id(), the BPF prog is -++ * effectively publicly exposed. However, retrieving via -++ * bpf_prog_get_fd_by_id() will take another reference, -++ * therefore it cannot be gone underneath us. -++ * -++ * Only for the time /after/ successful bpf_prog_new_fd() -++ * and before returning to userspace, we might just hold -++ * one reference and any parallel close on that fd could -++ * rip everything out. Hence, below notifications must -++ * happen before bpf_prog_new_fd(). -++ * -++ * Also, any failure handling from this point onwards must -++ * be using bpf_prog_put() given the program is exposed. -++ */ -++ bpf_prog_kallsyms_add(prog); -++ -++ err = bpf_prog_new_fd(prog); -++ if (err < 0) -++ bpf_prog_put(prog); -+ return err; -+ -+ free_used_maps: -+- free_used_maps(prog->aux); -++ /* In case we have subprogs, we need to wait for a grace -++ * period before we can tear down JIT memory since symbols -++ * are already exposed under kallsyms. -++ */ -++ __bpf_prog_put_noref(prog, prog->aux->func_cnt); -++ return err; -+ free_prog: -+ bpf_prog_uncharge_memlock(prog); -+-free_prog_nouncharge: -++free_prog_sec: -+ bpf_prog_free(prog); -+ return err; -+ } -+ -+-#define BPF_OBJ_LAST_FIELD bpf_fd -++#define BPF_OBJ_LAST_FIELD file_flags -+ -+ static int bpf_obj_pin(const union bpf_attr *attr) -+ { -+- if (CHECK_ATTR(BPF_OBJ)) -++ if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0) -+ return -EINVAL; -+ -+- return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname)); -++ return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname)); -+ } -+ -+ static int bpf_obj_get(const union bpf_attr *attr) -+ { -+- if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0) -++ if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 || -++ attr->file_flags & ~BPF_OBJ_FLAG_MASK) -+ return -EINVAL; -+ -+- return bpf_obj_get_user(u64_to_ptr(attr->pathname)); -++ return bpf_obj_get_user(u64_to_user_ptr(attr->pathname), -++ attr->file_flags); -+ } -+ -+-SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) -++ -++#define BPF_PROG_ATTACH_LAST_FIELD attach_flags -++ -++#define BPF_F_ATTACH_MASK \ -++ (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI) -++ -++ -++#define BPF_PROG_DETACH_LAST_FIELD attach_type -++ -++ -++#define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt -++ -++ -++#define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out -++ -++static int bpf_prog_test_run(const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -+ { -+- union bpf_attr attr = {}; -+- int err; -++ struct bpf_prog *prog; -++ int ret = -ENOTSUPP; -+ -+- if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled) -++ if (!capable(CAP_SYS_ADMIN)) -+ return -EPERM; -++ if (CHECK_ATTR(BPF_PROG_TEST_RUN)) -++ return -EINVAL; -++ -++ if ((attr->test.ctx_size_in && !attr->test.ctx_in) || -++ (!attr->test.ctx_size_in && attr->test.ctx_in)) -++ return -EINVAL; -++ -++ if ((attr->test.ctx_size_out && !attr->test.ctx_out) || -++ (!attr->test.ctx_size_out && attr->test.ctx_out)) -++ return -EINVAL; -++ -++ prog = bpf_prog_get(attr->test.prog_fd); -++ if (IS_ERR(prog)) -++ return PTR_ERR(prog); -++ -++ if (prog->aux->ops->test_run) -++ ret = prog->aux->ops->test_run(prog, attr, uattr); -++ -++ bpf_prog_put(prog); -++ return ret; -++} -++ -++#define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id -++ -++static int bpf_obj_get_next_id(const union bpf_attr *attr, -++ union bpf_attr __user *uattr, -++ struct idr *idr, -++ spinlock_t *lock) -++{ -++ u32 next_id = attr->start_id; -++ int err = 0; -++ -++ if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX) -++ return -EINVAL; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ next_id++; -++ spin_lock_bh(lock); -++ if (!idr_get_next(idr, &next_id)) -++ err = -ENOENT; -++ spin_unlock_bh(lock); -++ -++ if (!err) -++ err = put_user(next_id, &uattr->next_id); -++ -++ return err; -++} -++ -++#define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id -++ -++static int bpf_prog_get_fd_by_id(const union bpf_attr *attr) -++{ -++ struct bpf_prog *prog; -++ u32 id = attr->prog_id; -++ int fd; -++ -++ if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID)) -++ return -EINVAL; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ spin_lock_bh(&prog_idr_lock); -++ prog = idr_find(&prog_idr, id); -++ if (prog) -++ prog = bpf_prog_inc_not_zero(prog); -++ else -++ prog = ERR_PTR(-ENOENT); -++ spin_unlock_bh(&prog_idr_lock); -++ -++ if (IS_ERR(prog)) -++ return PTR_ERR(prog); -++ -++ fd = bpf_prog_new_fd(prog); -++ if (fd < 0) -++ bpf_prog_put(prog); -++ -++ return fd; -++} -++ -++#define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags -++ -++static int bpf_map_get_fd_by_id(const union bpf_attr *attr) -++{ -++ struct bpf_map *map; -++ u32 id = attr->map_id; -++ int f_flags; -++ int fd; -++ -++ if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) || -++ attr->open_flags & ~BPF_OBJ_FLAG_MASK) -++ return -EINVAL; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ f_flags = bpf_get_file_flag(attr->open_flags); -++ if (f_flags < 0) -++ return f_flags; -++ -++ spin_lock_bh(&map_idr_lock); -++ map = idr_find(&map_idr, id); -++ if (map) -++ map = __bpf_map_inc_not_zero(map, true); -++ else -++ map = ERR_PTR(-ENOENT); -++ spin_unlock_bh(&map_idr_lock); -++ -++ if (IS_ERR(map)) -++ return PTR_ERR(map); -+ -+- if (!access_ok(VERIFY_READ, uattr, 1)) -++ fd = bpf_map_new_fd(map, f_flags); -++ if (fd < 0) -++ bpf_map_put_with_uref(map); -++ -++ return fd; -++} -++ -++static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, -++ unsigned long addr, u32 *off, -++ u32 *type) -++{ -++ const struct bpf_map *map; -++ int i; -++ -++ for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { -++ map = prog->aux->used_maps[i]; -++ if (map == (void *)addr) { -++ *type = BPF_PSEUDO_MAP_FD; -++ return map; -++ } -++ if (!map->ops->map_direct_value_meta) -++ continue; -++ if (!map->ops->map_direct_value_meta(map, addr, off)) { -++ *type = BPF_PSEUDO_MAP_VALUE; -++ return map; -++ } -++ } -++ -++ return NULL; -++} -++ -++static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, -++ const struct cred *f_cred) -++{ -++ const struct bpf_map *map; -++ struct bpf_insn *insns; -++ u32 off, type; -++ u64 imm; -++ int i; -++ -++ insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog), -++ GFP_USER); -++ if (!insns) -++ return insns; -++ -++ for (i = 0; i < prog->len; i++) { -++ if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) { -++ insns[i].code = BPF_JMP | BPF_CALL; -++ insns[i].imm = BPF_FUNC_tail_call; -++ /* fall-through */ -++ } -++ if (insns[i].code == (BPF_JMP | BPF_CALL) || -++ insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) { -++ if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) -++ insns[i].code = BPF_JMP | BPF_CALL; -++ if (!bpf_dump_raw_ok(f_cred)) -++ insns[i].imm = 0; -++ continue; -++ } -++ -++ if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW)) -++ continue; -++ -++ imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; -++ map = bpf_map_from_imm(prog, imm, &off, &type); -++ if (map) { -++ insns[i].src_reg = type; -++ insns[i].imm = map->id; -++ insns[i + 1].imm = off; -++ continue; -++ } -++ } -++ -++ return insns; -++} -++ -++static int set_info_rec_size(struct bpf_prog_info *info) -++{ -++ /* -++ * Ensure info.*_rec_size is the same as kernel expected size -++ * -++ * or -++ * -++ * Only allow zero *_rec_size if both _rec_size and _cnt are -++ * zero. In this case, the kernel will set the expected -++ * _rec_size back to the info. -++ */ -++ -++ if ((info->nr_func_info || info->func_info_rec_size) && -++ info->func_info_rec_size != sizeof(struct bpf_func_info)) -++ return -EINVAL; -++ -++ if ((info->nr_line_info || info->line_info_rec_size) && -++ info->line_info_rec_size != sizeof(struct bpf_line_info)) -++ return -EINVAL; -++ -++ if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && -++ info->jited_line_info_rec_size != sizeof(__u64)) -++ return -EINVAL; -++ -++ info->func_info_rec_size = sizeof(struct bpf_func_info); -++ info->line_info_rec_size = sizeof(struct bpf_line_info); -++ info->jited_line_info_rec_size = sizeof(__u64); -++ -++ return 0; -++} -++ -++static int bpf_prog_get_info_by_fd(struct file *file, -++ struct bpf_prog *prog, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); -++ struct bpf_prog_info info; -++ u32 info_len = attr->info.info_len; -++ struct bpf_prog_stats stats; -++ char __user *uinsns; -++ u32 ulen; -++ int err; -++ -++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); -++ if (err) -++ return err; -++ info_len = min_t(u32, sizeof(info), info_len); -++ -++ memset(&info, 0, sizeof(info)); -++ if (copy_from_user(&info, uinfo, info_len)) -+ return -EFAULT; -+ -+- if (size > PAGE_SIZE) /* silly large */ -+- return -E2BIG; -++ info.type = prog->type; -++ info.id = prog->aux->id; -++ info.load_time = prog->aux->load_time; -++ info.created_by_uid = from_kuid_munged(current_user_ns(), -++ prog->aux->user->uid); -++ info.gpl_compatible = prog->gpl_compatible; -++ -++ memcpy(info.tag, prog->tag, sizeof(prog->tag)); -++ memcpy(info.name, prog->aux->name, sizeof(prog->aux->name)); -++ -++ ulen = info.nr_map_ids; -++ info.nr_map_ids = prog->aux->used_map_cnt; -++ ulen = min_t(u32, info.nr_map_ids, ulen); -++ if (ulen) { -++ u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids); -++ u32 i; -++ -++ for (i = 0; i < ulen; i++) -++ if (put_user(prog->aux->used_maps[i]->id, -++ &user_map_ids[i])) -++ return -EFAULT; -++ } -++ -++ err = set_info_rec_size(&info); -++ if (err) -++ return err; -++ -++ bpf_prog_get_stats(prog, &stats); -++ info.run_time_ns = stats.nsecs; -++ info.run_cnt = stats.cnt; -++ -++ if (!capable(CAP_SYS_ADMIN)) { -++ info.jited_prog_len = 0; -++ info.xlated_prog_len = 0; -++ info.nr_jited_ksyms = 0; -++ info.nr_jited_func_lens = 0; -++ info.nr_func_info = 0; -++ info.nr_line_info = 0; -++ info.nr_jited_line_info = 0; -++ goto done; -++ } -++ -++ ulen = info.xlated_prog_len; -++ info.xlated_prog_len = bpf_prog_insn_size(prog); -++ if (info.xlated_prog_len && ulen) { -++ struct bpf_insn *insns_sanitized; -++ bool fault; -++ -++ if (prog->blinded && !bpf_dump_raw_ok(file->f_cred)) { -++ info.xlated_prog_insns = 0; -++ goto done; -++ } -++ insns_sanitized = bpf_insn_prepare_dump(prog, file->f_cred); -++ if (!insns_sanitized) -++ return -ENOMEM; -++ uinsns = u64_to_user_ptr(info.xlated_prog_insns); -++ ulen = min_t(u32, info.xlated_prog_len, ulen); -++ fault = copy_to_user(uinsns, insns_sanitized, ulen); -++ kfree(insns_sanitized); -++ if (fault) -++ return -EFAULT; -++ } -++ -++ /* NOTE: the following code is supposed to be skipped for offload. -++ * bpf_prog_offload_info_fill() is the place to fill similar fields -++ * for offload. -++ */ -++ ulen = info.jited_prog_len; -++ if (prog->aux->func_cnt) { -++ u32 i; -++ -++ info.jited_prog_len = 0; -++ for (i = 0; i < prog->aux->func_cnt; i++) -++ info.jited_prog_len += prog->aux->func[i]->jited_len; -++ } else { -++ info.jited_prog_len = prog->jited_len; -++ } -++ -++ if (info.jited_prog_len && ulen) { -++ if (bpf_dump_raw_ok(file->f_cred)) { -++ uinsns = u64_to_user_ptr(info.jited_prog_insns); -++ ulen = min_t(u32, info.jited_prog_len, ulen); -++ -++ /* for multi-function programs, copy the JITed -++ * instructions for all the functions -++ */ -++ if (prog->aux->func_cnt) { -++ u32 len, free, i; -++ u8 *img; -++ -++ free = ulen; -++ for (i = 0; i < prog->aux->func_cnt; i++) { -++ len = prog->aux->func[i]->jited_len; -++ len = min_t(u32, len, free); -++ img = (u8 *) prog->aux->func[i]->bpf_func; -++ if (copy_to_user(uinsns, img, len)) -++ return -EFAULT; -++ uinsns += len; -++ free -= len; -++ if (!free) -++ break; -++ } -++ } else { -++ if (copy_to_user(uinsns, prog->bpf_func, ulen)) -++ return -EFAULT; -++ } -++ } else { -++ info.jited_prog_insns = 0; -++ } -++ } -++ -++ ulen = info.nr_jited_ksyms; -++ info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; -++ if (ulen) { -++ if (bpf_dump_raw_ok(file->f_cred)) { -++ unsigned long ksym_addr; -++ u64 __user *user_ksyms; -++ u32 i; -++ -++ /* copy the address of the kernel symbol -++ * corresponding to each function -++ */ -++ ulen = min_t(u32, info.nr_jited_ksyms, ulen); -++ user_ksyms = u64_to_user_ptr(info.jited_ksyms); -++ if (prog->aux->func_cnt) { -++ for (i = 0; i < ulen; i++) { -++ ksym_addr = (unsigned long) -++ prog->aux->func[i]->bpf_func; -++ if (put_user((u64) ksym_addr, -++ &user_ksyms[i])) -++ return -EFAULT; -++ } -++ } else { -++ ksym_addr = (unsigned long) prog->bpf_func; -++ if (put_user((u64) ksym_addr, &user_ksyms[0])) -++ return -EFAULT; -++ } -++ } else { -++ info.jited_ksyms = 0; -++ } -++ } -++ -++ ulen = info.nr_jited_func_lens; -++ info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; -++ if (ulen) { -++ if (bpf_dump_raw_ok(file->f_cred)) { -++ u32 __user *user_lens; -++ u32 func_len, i; -++ -++ /* copy the JITed image lengths for each function */ -++ ulen = min_t(u32, info.nr_jited_func_lens, ulen); -++ user_lens = u64_to_user_ptr(info.jited_func_lens); -++ if (prog->aux->func_cnt) { -++ for (i = 0; i < ulen; i++) { -++ func_len = -++ prog->aux->func[i]->jited_len; -++ if (put_user(func_len, &user_lens[i])) -++ return -EFAULT; -++ } -++ } else { -++ func_len = prog->jited_len; -++ if (put_user(func_len, &user_lens[0])) -++ return -EFAULT; -++ } -++ } else { -++ info.jited_func_lens = 0; -++ } -++ } -++ -++ if (prog->aux->btf) -++ info.btf_id = btf_id(prog->aux->btf); -++ -++ ulen = info.nr_func_info; -++ info.nr_func_info = prog->aux->func_info_cnt; -++ if (info.nr_func_info && ulen) { -++ char __user *user_finfo; -++ -++ user_finfo = u64_to_user_ptr(info.func_info); -++ ulen = min_t(u32, info.nr_func_info, ulen); -++ if (copy_to_user(user_finfo, prog->aux->func_info, -++ info.func_info_rec_size * ulen)) -++ return -EFAULT; -++ } -++ -++ ulen = info.nr_line_info; -++ info.nr_line_info = prog->aux->nr_linfo; -++ if (info.nr_line_info && ulen) { -++ __u8 __user *user_linfo; -++ -++ user_linfo = u64_to_user_ptr(info.line_info); -++ ulen = min_t(u32, info.nr_line_info, ulen); -++ if (copy_to_user(user_linfo, prog->aux->linfo, -++ info.line_info_rec_size * ulen)) -++ return -EFAULT; -++ } -+ -+- /* If we're handed a bigger struct than we know of, -+- * ensure all the unknown bits are 0 - i.e. new -+- * user-space does not rely on any kernel feature -+- * extensions we dont know about yet. -+- */ -+- if (size > sizeof(attr)) { -+- unsigned char __user *addr; -+- unsigned char __user *end; -+- unsigned char val; -+- -+- addr = (void __user *)uattr + sizeof(attr); -+- end = (void __user *)uattr + size; -+- -+- for (; addr < end; addr++) { -+- err = get_user(val, addr); -+- if (err) -+- return err; -+- if (val) -+- return -E2BIG; -++ ulen = info.nr_jited_line_info; -++ if (prog->aux->jited_linfo) -++ info.nr_jited_line_info = prog->aux->nr_linfo; -++ else -++ info.nr_jited_line_info = 0; -++ if (info.nr_jited_line_info && ulen) { -++ if (bpf_dump_raw_ok(file->f_cred)) { -++ __u64 __user *user_linfo; -++ u32 i; -++ -++ user_linfo = u64_to_user_ptr(info.jited_line_info); -++ ulen = min_t(u32, info.nr_jited_line_info, ulen); -++ for (i = 0; i < ulen; i++) { -++ if (put_user((__u64)(long)prog->aux->jited_linfo[i], -++ &user_linfo[i])) -++ return -EFAULT; -++ } -++ } else { -++ info.jited_line_info = 0; -+ } -+- size = sizeof(attr); -+ } -+ -++ ulen = info.nr_prog_tags; -++ info.nr_prog_tags = prog->aux->func_cnt ? : 1; -++ if (ulen) { -++ __u8 __user (*user_prog_tags)[BPF_TAG_SIZE]; -++ u32 i; -++ -++ user_prog_tags = u64_to_user_ptr(info.prog_tags); -++ ulen = min_t(u32, info.nr_prog_tags, ulen); -++ if (prog->aux->func_cnt) { -++ for (i = 0; i < ulen; i++) { -++ if (copy_to_user(user_prog_tags[i], -++ prog->aux->func[i]->tag, -++ BPF_TAG_SIZE)) -++ return -EFAULT; -++ } -++ } else { -++ if (copy_to_user(user_prog_tags[0], -++ prog->tag, BPF_TAG_SIZE)) -++ return -EFAULT; -++ } -++ } -++ -++done: -++ if (copy_to_user(uinfo, &info, info_len) || -++ put_user(info_len, &uattr->info.info_len)) -++ return -EFAULT; -++ -++ return 0; -++} -++ -++static int bpf_map_get_info_by_fd(struct file *file, -++ struct bpf_map *map, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); -++ struct bpf_map_info info; -++ u32 info_len = attr->info.info_len; -++ int err; -++ -++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len); -++ if (err) -++ return err; -++ info_len = min_t(u32, sizeof(info), info_len); -++ -++ memset(&info, 0, sizeof(info)); -++ info.type = map->map_type; -++ info.id = map->id; -++ info.key_size = map->key_size; -++ info.value_size = map->value_size; -++ info.max_entries = map->max_entries; -++ info.map_flags = map->map_flags; -++ memcpy(info.name, map->name, sizeof(map->name)); -++ -++ if (map->btf) { -++ info.btf_id = btf_id(map->btf); -++ info.btf_key_type_id = map->btf_key_type_id; -++ info.btf_value_type_id = map->btf_value_type_id; -++ } -++ -++ if (copy_to_user(uinfo, &info, info_len) || -++ put_user(info_len, &uattr->info.info_len)) -++ return -EFAULT; -++ -++ return 0; -++} -++ -++static int bpf_btf_get_info_by_fd(struct file *file, -++ struct btf *btf, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info); -++ u32 info_len = attr->info.info_len; -++ int err; -++ -++ err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len); -++ if (err) -++ return err; -++ -++ return btf_get_info_by_fd(btf, attr, uattr); -++} -++ -++#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info -++ -++static int bpf_obj_get_info_by_fd(const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ int ufd = attr->info.bpf_fd; -++ struct fd f; -++ int err; -++ -++ if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD)) -++ return -EINVAL; -++ -++ f = fdget(ufd); -++ if (!f.file) -++ return -EBADFD; -++ -++ if (f.file->f_op == &bpf_prog_fops) -++ err = bpf_prog_get_info_by_fd(f.file, f.file->private_data, attr, -++ uattr); -++ else if (f.file->f_op == &bpf_map_fops) -++ err = bpf_map_get_info_by_fd(f.file, f.file->private_data, attr, -++ uattr); -++ else if (f.file->f_op == &btf_fops) -++ err = bpf_btf_get_info_by_fd(f.file, f.file->private_data, attr, uattr); -++ else -++ err = -EINVAL; -++ -++ fdput(f); -++ return err; -++} -++ -++#define BPF_BTF_LOAD_LAST_FIELD btf_log_level -++ -++static int bpf_btf_load(const union bpf_attr *attr) -++{ -++ if (CHECK_ATTR(BPF_BTF_LOAD)) -++ return -EINVAL; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ return btf_new_fd(attr); -++} -++ -++#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id -++ -++static int bpf_btf_get_fd_by_id(const union bpf_attr *attr) -++{ -++ if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID)) -++ return -EINVAL; -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ return btf_get_fd_by_id(attr->btf_id); -++} -++ -++ -++#define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr -++ -++SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) -++{ -++ union bpf_attr attr; -++ int err; -++ -++ if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) -++ return -EPERM; -++ -++ err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size); -++ if (err) -++ return err; -++ size = min_t(u32, size, sizeof(attr)); -++ -+ /* copy attributes from user space, may be less than sizeof(bpf_attr) */ -++ memset(&attr, 0, sizeof(attr)); -+ if (copy_from_user(&attr, uattr, size) != 0) -+ return -EFAULT; -+ -+@@ -779,8 +2364,11 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf -+ case BPF_MAP_GET_NEXT_KEY: -+ err = map_get_next_key(&attr); -+ break; -++ case BPF_MAP_FREEZE: -++ err = map_freeze(&attr); -++ break; -+ case BPF_PROG_LOAD: -+- err = bpf_prog_load(&attr); -++ err = bpf_prog_load(&attr, uattr); -+ break; -+ case BPF_OBJ_PIN: -+ err = bpf_obj_pin(&attr); -+@@ -788,6 +2376,39 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf -+ case BPF_OBJ_GET: -+ err = bpf_obj_get(&attr); -+ break; -++ case BPF_PROG_TEST_RUN: -++ err = bpf_prog_test_run(&attr, uattr); -++ break; -++ case BPF_PROG_GET_NEXT_ID: -++ err = bpf_obj_get_next_id(&attr, uattr, -++ &prog_idr, &prog_idr_lock); -++ break; -++ case BPF_MAP_GET_NEXT_ID: -++ err = bpf_obj_get_next_id(&attr, uattr, -++ &map_idr, &map_idr_lock); -++ break; -++ case BPF_BTF_GET_NEXT_ID: -++ err = bpf_obj_get_next_id(&attr, uattr, -++ &btf_idr, &btf_idr_lock); -++ break; -++ case BPF_PROG_GET_FD_BY_ID: -++ err = bpf_prog_get_fd_by_id(&attr); -++ break; -++ case BPF_MAP_GET_FD_BY_ID: -++ err = bpf_map_get_fd_by_id(&attr); -++ break; -++ case BPF_OBJ_GET_INFO_BY_FD: -++ err = bpf_obj_get_info_by_fd(&attr, uattr); -++ break; -++ case BPF_BTF_LOAD: -++ err = bpf_btf_load(&attr); -++ break; -++ case BPF_BTF_GET_FD_BY_ID: -++ err = bpf_btf_get_fd_by_id(&attr); -++ break; -++ case BPF_MAP_LOOKUP_AND_DELETE_ELEM: -++ err = map_lookup_and_delete_elem(&attr); -++ break; -+ default: -+ err = -EINVAL; -+ break; -+--- /dev/null -++++ b/kernel/bpf/sysfs_btf.c -+@@ -0,0 +1,45 @@ -++// SPDX-License-Identifier: GPL-2.0 -++/* -++ * Provide kernel BTF information for introspection and use by eBPF tools. -++ */ -++#include -++#include -++#include -++#include -++#include -++ -++/* See scripts/link-vmlinux.sh, gen_btf() func for details */ -++extern char __weak __start_BTF[]; -++extern char __weak __stop_BTF[]; -++ -++static ssize_t -++btf_vmlinux_read(struct file *file, struct kobject *kobj, -++ struct bin_attribute *bin_attr, -++ char *buf, loff_t off, size_t len) -++{ -++ memcpy(buf, __start_BTF + off, len); -++ return len; -++} -++ -++static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = { -++ .attr = { .name = "vmlinux", .mode = 0444, }, -++ .read = btf_vmlinux_read, -++}; -++ -++static struct kobject *btf_kobj; -++ -++static int __init btf_vmlinux_init(void) -++{ -++ bin_attr_btf_vmlinux.size = __stop_BTF - __start_BTF; -++ -++ if (!__start_BTF || bin_attr_btf_vmlinux.size == 0) -++ return 0; -++ -++ btf_kobj = kobject_create_and_add("btf", kernel_kobj); -++ if (!btf_kobj) -++ return -ENOMEM; -++ -++ return sysfs_create_bin_file(btf_kobj, &bin_attr_btf_vmlinux); -++} -++ -++subsys_initcall(btf_vmlinux_init); -+--- /dev/null -++++ b/kernel/bpf/tnum.c -+@@ -0,0 +1,196 @@ -++// SPDX-License-Identifier: GPL-2.0-only -++/* tnum: tracked (or tristate) numbers -++ * -++ * A tnum tracks knowledge about the bits of a value. Each bit can be either -++ * known (0 or 1), or unknown (x). Arithmetic operations on tnums will -++ * propagate the unknown bits such that the tnum result represents all the -++ * possible results for possible values of the operands. -++ */ -++#include -++#include -++ -++#define TNUM(_v, _m) (struct tnum){.value = _v, .mask = _m} -++/* A completely unknown value */ -++const struct tnum tnum_unknown = { .value = 0, .mask = -1 }; -++ -++struct tnum tnum_const(u64 value) -++{ -++ return TNUM(value, 0); -++} -++ -++struct tnum tnum_range(u64 min, u64 max) -++{ -++ u64 chi = min ^ max, delta; -++ u8 bits = fls64(chi); -++ -++ /* special case, needed because 1ULL << 64 is undefined */ -++ if (bits > 63) -++ return tnum_unknown; -++ /* e.g. if chi = 4, bits = 3, delta = (1<<3) - 1 = 7. -++ * if chi = 0, bits = 0, delta = (1<<0) - 1 = 0, so we return -++ * constant min (since min == max). -++ */ -++ delta = (1ULL << bits) - 1; -++ return TNUM(min & ~delta, delta); -++} -++ -++struct tnum tnum_lshift(struct tnum a, u8 shift) -++{ -++ return TNUM(a.value << shift, a.mask << shift); -++} -++ -++struct tnum tnum_rshift(struct tnum a, u8 shift) -++{ -++ return TNUM(a.value >> shift, a.mask >> shift); -++} -++ -++struct tnum tnum_arshift(struct tnum a, u8 min_shift, u8 insn_bitness) -++{ -++ /* if a.value is negative, arithmetic shifting by minimum shift -++ * will have larger negative offset compared to more shifting. -++ * If a.value is nonnegative, arithmetic shifting by minimum shift -++ * will have larger positive offset compare to more shifting. -++ */ -++ if (insn_bitness == 32) -++ return TNUM((u32)(((s32)a.value) >> min_shift), -++ (u32)(((s32)a.mask) >> min_shift)); -++ else -++ return TNUM((s64)a.value >> min_shift, -++ (s64)a.mask >> min_shift); -++} -++ -++struct tnum tnum_add(struct tnum a, struct tnum b) -++{ -++ u64 sm, sv, sigma, chi, mu; -++ -++ sm = a.mask + b.mask; -++ sv = a.value + b.value; -++ sigma = sm + sv; -++ chi = sigma ^ sv; -++ mu = chi | a.mask | b.mask; -++ return TNUM(sv & ~mu, mu); -++} -++ -++struct tnum tnum_sub(struct tnum a, struct tnum b) -++{ -++ u64 dv, alpha, beta, chi, mu; -++ -++ dv = a.value - b.value; -++ alpha = dv + a.mask; -++ beta = dv - b.mask; -++ chi = alpha ^ beta; -++ mu = chi | a.mask | b.mask; -++ return TNUM(dv & ~mu, mu); -++} -++ -++struct tnum tnum_and(struct tnum a, struct tnum b) -++{ -++ u64 alpha, beta, v; -++ -++ alpha = a.value | a.mask; -++ beta = b.value | b.mask; -++ v = a.value & b.value; -++ return TNUM(v, alpha & beta & ~v); -++} -++ -++struct tnum tnum_or(struct tnum a, struct tnum b) -++{ -++ u64 v, mu; -++ -++ v = a.value | b.value; -++ mu = a.mask | b.mask; -++ return TNUM(v, mu & ~v); -++} -++ -++struct tnum tnum_xor(struct tnum a, struct tnum b) -++{ -++ u64 v, mu; -++ -++ v = a.value ^ b.value; -++ mu = a.mask | b.mask; -++ return TNUM(v & ~mu, mu); -++} -++ -++/* half-multiply add: acc += (unknown * mask * value). -++ * An intermediate step in the multiply algorithm. -++ */ -++static struct tnum hma(struct tnum acc, u64 value, u64 mask) -++{ -++ while (mask) { -++ if (mask & 1) -++ acc = tnum_add(acc, TNUM(0, value)); -++ mask >>= 1; -++ value <<= 1; -++ } -++ return acc; -++} -++ -++struct tnum tnum_mul(struct tnum a, struct tnum b) -++{ -++ struct tnum acc; -++ u64 pi; -++ -++ pi = a.value * b.value; -++ acc = hma(TNUM(pi, 0), a.mask, b.mask | b.value); -++ return hma(acc, b.mask, a.value); -++} -++ -++/* Note that if a and b disagree - i.e. one has a 'known 1' where the other has -++ * a 'known 0' - this will return a 'known 1' for that bit. -++ */ -++struct tnum tnum_intersect(struct tnum a, struct tnum b) -++{ -++ u64 v, mu; -++ -++ v = a.value | b.value; -++ mu = a.mask & b.mask; -++ return TNUM(v & ~mu, mu); -++} -++ -++struct tnum tnum_cast(struct tnum a, u8 size) -++{ -++ a.value &= (1ULL << (size * 8)) - 1; -++ a.mask &= (1ULL << (size * 8)) - 1; -++ return a; -++} -++ -++bool tnum_is_aligned(struct tnum a, u64 size) -++{ -++ if (!size) -++ return true; -++ return !((a.value | a.mask) & (size - 1)); -++} -++ -++bool tnum_in(struct tnum a, struct tnum b) -++{ -++ if (b.mask & ~a.mask) -++ return false; -++ b.value &= ~a.mask; -++ return a.value == b.value; -++} -++ -++int tnum_strn(char *str, size_t size, struct tnum a) -++{ -++ return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask); -++} -++EXPORT_SYMBOL_GPL(tnum_strn); -++ -++int tnum_sbin(char *str, size_t size, struct tnum a) -++{ -++ size_t n; -++ -++ for (n = 64; n; n--) { -++ if (n < size) { -++ if (a.mask & 1) -++ str[n - 1] = 'x'; -++ else if (a.value & 1) -++ str[n - 1] = '1'; -++ else -++ str[n - 1] = '0'; -++ } -++ a.mask >>= 1; -++ a.value >>= 1; -++ } -++ str[min(size - 1, (size_t)64)] = 0; -++ return 64; -++} -+--- a/kernel/bpf/verifier.c -++++ b/kernel/bpf/verifier.c -+@@ -1,22 +1,36 @@ -++// SPDX-License-Identifier: GPL-2.0-only -+ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com -+- * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of version 2 of the GNU General Public -+- * License as published by the Free Software Foundation. -+- * -+- * This program is distributed in the hope that it will be useful, but -+- * WITHOUT ANY WARRANTY; without even the implied warranty of -+- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -+- * General Public License for more details. -++ * Copyright (c) 2016 Facebook -++ * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io -+ */ -++#include -+ #include -+ #include -+ #include -+ #include -++#include -++#include -+ #include -+ #include -+ #include -+ #include -++#include -++#include -++#include -++#include -++#include -++#include -++ -++#include "disasm.h" -++ -++static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { -++#define BPF_PROG_TYPE(_id, _name) \ -++ [_id] = & _name ## _verifier_ops, -++#define BPF_MAP_TYPE(_id, _ops) -++#include -++#undef BPF_PROG_TYPE -++#undef BPF_MAP_TYPE -++}; -+ -+ /* bpf_check() is a static code analyzer that walks eBPF program -+ * instruction by instruction and updates register/stack state. -+@@ -30,7 +44,7 @@ -+ * - out of bounds or malformed jumps -+ * The second pass is all possible path descent from the 1st insn. -+ * Since it's analyzing all pathes through the program, the length of the -+- * analysis is limited to 32k insn, which may be hit even if total number of -++ * analysis is limited to 64k insn, which may be hit even if total number of -+ * insn is less then 4K, but there are too many branches that change stack/regs. -+ * Number of 'branches to be analyzed' is limited to 1k -+ * -+@@ -58,13 +72,13 @@ -+ * (and -20 constant is saved for further stack bounds checking). -+ * Meaning that this reg is a pointer to stack plus known immediate constant. -+ * -+- * Most of the time the registers have UNKNOWN_VALUE type, which -++ * Most of the time the registers have SCALAR_VALUE type, which -+ * means the register has some value, but it's not a valid pointer. -+- * (like pointer plus pointer becomes UNKNOWN_VALUE type) -++ * (like pointer plus pointer becomes SCALAR_VALUE type) -+ * -+ * When verifier sees load or store instructions the type of base register -+- * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, FRAME_PTR. These are three pointer -+- * types recognized by check_mem_access() function. -++ * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are -++ * four pointer types recognized by check_mem_access() function. -+ * -+ * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' -+ * and the range of [ptr, ptr + map's value_size) is accessible. -+@@ -123,346 +137,713 @@ -+ * -+ * After the call R0 is set to return type of the function and registers R1-R5 -+ * are set to NOT_INIT to indicate that they are no longer readable. -++ * -++ * The following reference types represent a potential reference to a kernel -++ * resource which, after first being allocated, must be checked and freed by -++ * the BPF program: -++ * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET -++ * -++ * When the verifier sees a helper call return a reference type, it allocates a -++ * pointer id for the reference and stores it in the current function state. -++ * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into -++ * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type -++ * passes through a NULL-check conditional. For the branch wherein the state is -++ * changed to CONST_IMM, the verifier releases the reference. -++ * -++ * For each helper function that allocates a reference, such as -++ * bpf_sk_lookup_tcp(), there is a corresponding release function, such as -++ * bpf_sk_release(). When a reference type passes into the release function, -++ * the verifier also releases the reference. If any unchecked or unreleased -++ * reference remains at the end of the program, the verifier rejects it. -+ */ -+ -+-/* types of values stored in eBPF registers */ -+-enum bpf_reg_type { -+- NOT_INIT = 0, /* nothing was written into register */ -+- UNKNOWN_VALUE, /* reg doesn't contain a valid pointer */ -+- PTR_TO_CTX, /* reg points to bpf_context */ -+- CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ -+- PTR_TO_MAP_VALUE, /* reg points to map element value */ -+- PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ -+- FRAME_PTR, /* reg == frame_pointer */ -+- PTR_TO_STACK, /* reg == frame_pointer + imm */ -+- CONST_IMM, /* constant integer value */ -+-}; -+- -+-struct reg_state { -+- enum bpf_reg_type type; -+- union { -+- /* valid when type == CONST_IMM | PTR_TO_STACK */ -+- int imm; -+- -+- /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | -+- * PTR_TO_MAP_VALUE_OR_NULL -+- */ -+- struct bpf_map *map_ptr; -+- }; -+-}; -+- -+-enum bpf_stack_slot_type { -+- STACK_INVALID, /* nothing was stored in this stack slot */ -+- STACK_SPILL, /* register spilled into stack */ -+- STACK_MISC /* BPF program wrote some data into this slot */ -+-}; -+- -+-#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ -+- -+-/* state of the program: -+- * type of all registers and stack info -+- */ -+-struct verifier_state { -+- struct reg_state regs[MAX_BPF_REG]; -+- u8 stack_slot_type[MAX_BPF_STACK]; -+- struct reg_state spilled_regs[MAX_BPF_STACK / BPF_REG_SIZE]; -+-}; -+- -+-/* linked list of verifier states used to prune search */ -+-struct verifier_state_list { -+- struct verifier_state state; -+- struct verifier_state_list *next; -+-}; -+- -+ /* verifier_state + insn_idx are pushed to stack when branch is encountered */ -+-struct verifier_stack_elem { -++struct bpf_verifier_stack_elem { -+ /* verifer state is 'st' -+ * before processing instruction 'insn_idx' -+ * and after processing instruction 'prev_insn_idx' -+ */ -+- struct verifier_state st; -++ struct bpf_verifier_state st; -+ int insn_idx; -+ int prev_insn_idx; -+- struct verifier_stack_elem *next; -++ struct bpf_verifier_stack_elem *next; -+ }; -+ -+-#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ -++#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 -++#define BPF_COMPLEXITY_LIMIT_STATES 64 -+ -+-/* single container for all structs -+- * one verifier_env per bpf_check() call -+- */ -+-struct verifier_env { -+- struct bpf_prog *prog; /* eBPF program being verified */ -+- struct verifier_stack_elem *head; /* stack of verifier states to be processed */ -+- int stack_size; /* number of states to be processed */ -+- struct verifier_state cur_state; /* current verifier state */ -+- struct verifier_state_list **explored_states; /* search pruning optimization */ -+- struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */ -+- u32 used_map_cnt; /* number of used maps */ -+- bool allow_ptr_leaks; -+-}; -++#define BPF_MAP_PTR_UNPRIV 1UL -++#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ -++ POISON_POINTER_DELTA)) -++#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) -+ -+-/* verbose verifier prints what it's seeing -+- * bpf_check() is called under lock, so no race to access these global vars -+- */ -+-static u32 log_level, log_size, log_len; -+-static char *log_buf; -++static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) -++{ -++ return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON; -++} -++ -++static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) -++{ -++ return aux->map_state & BPF_MAP_PTR_UNPRIV; -++} -++ -++static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, -++ const struct bpf_map *map, bool unpriv) -++{ -++ BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); -++ unpriv |= bpf_map_ptr_unpriv(aux); -++ aux->map_state = (unsigned long)map | -++ (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); -++} -++ -++struct bpf_call_arg_meta { -++ struct bpf_map *map_ptr; -++ bool raw_mode; -++ bool pkt_access; -++ int regno; -++ int access_size; -++ u64 msize_max_value; -++ int ref_obj_id; -++ int func_id; -++}; -+ -+ static DEFINE_MUTEX(bpf_verifier_lock); -+ -++static const struct bpf_line_info * -++find_linfo(const struct bpf_verifier_env *env, u32 insn_off) -++{ -++ const struct bpf_line_info *linfo; -++ const struct bpf_prog *prog; -++ u32 i, nr_linfo; -++ -++ prog = env->prog; -++ nr_linfo = prog->aux->nr_linfo; -++ -++ if (!nr_linfo || insn_off >= prog->len) -++ return NULL; -++ -++ linfo = prog->aux->linfo; -++ for (i = 1; i < nr_linfo; i++) -++ if (insn_off < linfo[i].insn_off) -++ break; -++ -++ return &linfo[i - 1]; -++} -++ -++void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, -++ va_list args) -++{ -++ unsigned int n; -++ -++ n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); -++ -++ WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, -++ "verifier log line truncated - local buffer too short\n"); -++ -++ n = min(log->len_total - log->len_used - 1, n); -++ log->kbuf[n] = '\0'; -++ -++ if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) -++ log->len_used += n; -++ else -++ log->ubuf = NULL; -++} -++ -+ /* log_level controls verbosity level of eBPF verifier. -+- * verbose() is used to dump the verification trace to the log, so the user -+- * can figure out what's wrong with the program -++ * bpf_verifier_log_write() is used to dump the verification trace to the log, -++ * so the user can figure out what's wrong with the program -+ */ -+-static __printf(1, 2) void verbose(const char *fmt, ...) -++__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, -++ const char *fmt, ...) -+ { -+ va_list args; -+ -+- if (log_level == 0 || log_len >= log_size - 1) -++ if (!bpf_verifier_log_needed(&env->log)) -+ return; -+ -+ va_start(args, fmt); -+- log_len += vscnprintf(log_buf + log_len, log_size - log_len, fmt, args); -++ bpf_verifier_vlog(&env->log, fmt, args); -+ va_end(args); -+ } -++EXPORT_SYMBOL_GPL(bpf_verifier_log_write); -++ -++__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) -++{ -++ struct bpf_verifier_env *env = private_data; -++ va_list args; -++ -++ if (!bpf_verifier_log_needed(&env->log)) -++ return; -++ -++ va_start(args, fmt); -++ bpf_verifier_vlog(&env->log, fmt, args); -++ va_end(args); -++} -++ -++static const char *ltrim(const char *s) -++{ -++ while (isspace(*s)) -++ s++; -++ -++ return s; -++} -++ -++__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, -++ u32 insn_off, -++ const char *prefix_fmt, ...) -++{ -++ const struct bpf_line_info *linfo; -++ -++ if (!bpf_verifier_log_needed(&env->log)) -++ return; -++ -++ linfo = find_linfo(env, insn_off); -++ if (!linfo || linfo == env->prev_linfo) -++ return; -++ -++ if (prefix_fmt) { -++ va_list args; -++ -++ va_start(args, prefix_fmt); -++ bpf_verifier_vlog(&env->log, prefix_fmt, args); -++ va_end(args); -++ } -++ -++ verbose(env, "%s\n", -++ ltrim(btf_name_by_offset(env->prog->aux->btf, -++ linfo->line_off))); -++ -++ env->prev_linfo = linfo; -++} -++ -++static bool type_is_pkt_pointer(enum bpf_reg_type type) -++{ -++ return type == PTR_TO_PACKET || -++ type == PTR_TO_PACKET_META; -++} -++ -++static bool type_is_sk_pointer(enum bpf_reg_type type) -++{ -++ return type == PTR_TO_SOCKET || -++ type == PTR_TO_SOCK_COMMON || -++ type == PTR_TO_TCP_SOCK || -++ type == PTR_TO_XDP_SOCK; -++} -++ -++static bool reg_type_may_be_null(enum bpf_reg_type type) -++{ -++ return type == PTR_TO_MAP_VALUE_OR_NULL || -++ type == PTR_TO_SOCKET_OR_NULL || -++ type == PTR_TO_SOCK_COMMON_OR_NULL || -++ type == PTR_TO_TCP_SOCK_OR_NULL; -++} -++ -++static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) -++{ -++ return reg->type == PTR_TO_MAP_VALUE && -++ map_value_has_spin_lock(reg->map_ptr); -++} -++ -++static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) -++{ -++ return type == PTR_TO_SOCKET || -++ type == PTR_TO_SOCKET_OR_NULL || -++ type == PTR_TO_TCP_SOCK || -++ type == PTR_TO_TCP_SOCK_OR_NULL; -++} -++ -++static bool arg_type_may_be_refcounted(enum bpf_arg_type type) -++{ -++ return type == ARG_PTR_TO_SOCK_COMMON; -++} -++ -++/* Determine whether the function releases some resources allocated by another -++ * function call. The first reference type argument will be assumed to be -++ * released by release_reference(). -++ */ -++static bool is_release_function(enum bpf_func_id func_id) -++{ -++ return func_id == BPF_FUNC_sk_release; -++} -++ -++static bool is_acquire_function(enum bpf_func_id func_id) -++{ -++ return func_id == BPF_FUNC_sk_lookup_tcp || -++ func_id == BPF_FUNC_sk_lookup_udp || -++ func_id == BPF_FUNC_skc_lookup_tcp; -++} -++ -++static bool is_ptr_cast_function(enum bpf_func_id func_id) -++{ -++ return func_id == BPF_FUNC_tcp_sock || -++ func_id == BPF_FUNC_sk_fullsock; -++} -+ -+ /* string representation of 'enum bpf_reg_type' */ -+ static const char * const reg_type_str[] = { -+ [NOT_INIT] = "?", -+- [UNKNOWN_VALUE] = "inv", -++ [SCALAR_VALUE] = "inv", -+ [PTR_TO_CTX] = "ctx", -+ [CONST_PTR_TO_MAP] = "map_ptr", -+ [PTR_TO_MAP_VALUE] = "map_value", -+ [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", -+- [FRAME_PTR] = "fp", -+ [PTR_TO_STACK] = "fp", -+- [CONST_IMM] = "imm", -++ [PTR_TO_PACKET] = "pkt", -++ [PTR_TO_PACKET_META] = "pkt_meta", -++ [PTR_TO_PACKET_END] = "pkt_end", -++ [PTR_TO_FLOW_KEYS] = "flow_keys", -++ [PTR_TO_SOCKET] = "sock", -++ [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", -++ [PTR_TO_SOCK_COMMON] = "sock_common", -++ [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", -++ [PTR_TO_TCP_SOCK] = "tcp_sock", -++ [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", -++ [PTR_TO_TP_BUFFER] = "tp_buffer", -++ [PTR_TO_XDP_SOCK] = "xdp_sock", -+ }; -+ -+-static void print_verifier_state(struct verifier_env *env) -++static char slot_type_char[] = { -++ [STACK_INVALID] = '?', -++ [STACK_SPILL] = 'r', -++ [STACK_MISC] = 'm', -++ [STACK_ZERO] = '0', -++}; -++ -++static void print_liveness(struct bpf_verifier_env *env, -++ enum bpf_reg_liveness live) -++{ -++ if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) -++ verbose(env, "_"); -++ if (live & REG_LIVE_READ) -++ verbose(env, "r"); -++ if (live & REG_LIVE_WRITTEN) -++ verbose(env, "w"); -++ if (live & REG_LIVE_DONE) -++ verbose(env, "D"); -++} -++ -++static struct bpf_func_state *func(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg) -+ { -++ struct bpf_verifier_state *cur = env->cur_state; -++ -++ return cur->frame[reg->frameno]; -++} -++ -++static void print_verifier_state(struct bpf_verifier_env *env, -++ const struct bpf_func_state *state) -++{ -++ const struct bpf_reg_state *reg; -+ enum bpf_reg_type t; -+ int i; -+ -++ if (state->frameno) -++ verbose(env, " frame%d:", state->frameno); -+ for (i = 0; i < MAX_BPF_REG; i++) { -+- t = env->cur_state.regs[i].type; -++ reg = &state->regs[i]; -++ t = reg->type; -+ if (t == NOT_INIT) -+ continue; -+- verbose(" R%d=%s", i, reg_type_str[t]); -+- if (t == CONST_IMM || t == PTR_TO_STACK) -+- verbose("%d", env->cur_state.regs[i].imm); -+- else if (t == CONST_PTR_TO_MAP || t == PTR_TO_MAP_VALUE || -+- t == PTR_TO_MAP_VALUE_OR_NULL) -+- verbose("(ks=%d,vs=%d)", -+- env->cur_state.regs[i].map_ptr->key_size, -+- env->cur_state.regs[i].map_ptr->value_size); -+- } -+- for (i = 0; i < MAX_BPF_STACK; i += BPF_REG_SIZE) { -+- if (env->cur_state.stack_slot_type[i] == STACK_SPILL) -+- verbose(" fp%d=%s", -MAX_BPF_STACK + i, -+- reg_type_str[env->cur_state.spilled_regs[i / BPF_REG_SIZE].type]); -+- } -+- verbose("\n"); -+-} -+- -+-static const char *const bpf_class_string[] = { -+- [BPF_LD] = "ld", -+- [BPF_LDX] = "ldx", -+- [BPF_ST] = "st", -+- [BPF_STX] = "stx", -+- [BPF_ALU] = "alu", -+- [BPF_JMP] = "jmp", -+- [BPF_RET] = "BUG", -+- [BPF_ALU64] = "alu64", -+-}; -++ verbose(env, " R%d", i); -++ print_liveness(env, reg->live); -++ verbose(env, "=%s", reg_type_str[t]); -++ if (t == SCALAR_VALUE && reg->precise) -++ verbose(env, "P"); -++ if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && -++ tnum_is_const(reg->var_off)) { -++ /* reg->off should be 0 for SCALAR_VALUE */ -++ verbose(env, "%lld", reg->var_off.value + reg->off); -++ } else { -++ verbose(env, "(id=%d", reg->id); -++ if (reg_type_may_be_refcounted_or_null(t)) -++ verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); -++ if (t != SCALAR_VALUE) -++ verbose(env, ",off=%d", reg->off); -++ if (type_is_pkt_pointer(t)) -++ verbose(env, ",r=%d", reg->range); -++ else if (t == CONST_PTR_TO_MAP || -++ t == PTR_TO_MAP_VALUE || -++ t == PTR_TO_MAP_VALUE_OR_NULL) -++ verbose(env, ",ks=%d,vs=%d", -++ reg->map_ptr->key_size, -++ reg->map_ptr->value_size); -++ if (tnum_is_const(reg->var_off)) { -++ /* Typically an immediate SCALAR_VALUE, but -++ * could be a pointer whose offset is too big -++ * for reg->off -++ */ -++ verbose(env, ",imm=%llx", reg->var_off.value); -++ } else { -++ if (reg->smin_value != reg->umin_value && -++ reg->smin_value != S64_MIN) -++ verbose(env, ",smin_value=%lld", -++ (long long)reg->smin_value); -++ if (reg->smax_value != reg->umax_value && -++ reg->smax_value != S64_MAX) -++ verbose(env, ",smax_value=%lld", -++ (long long)reg->smax_value); -++ if (reg->umin_value != 0) -++ verbose(env, ",umin_value=%llu", -++ (unsigned long long)reg->umin_value); -++ if (reg->umax_value != U64_MAX) -++ verbose(env, ",umax_value=%llu", -++ (unsigned long long)reg->umax_value); -++ if (!tnum_is_unknown(reg->var_off)) { -++ char tn_buf[48]; -+ -+-static const char *const bpf_alu_string[16] = { -+- [BPF_ADD >> 4] = "+=", -+- [BPF_SUB >> 4] = "-=", -+- [BPF_MUL >> 4] = "*=", -+- [BPF_DIV >> 4] = "/=", -+- [BPF_OR >> 4] = "|=", -+- [BPF_AND >> 4] = "&=", -+- [BPF_LSH >> 4] = "<<=", -+- [BPF_RSH >> 4] = ">>=", -+- [BPF_NEG >> 4] = "neg", -+- [BPF_MOD >> 4] = "%=", -+- [BPF_XOR >> 4] = "^=", -+- [BPF_MOV >> 4] = "=", -+- [BPF_ARSH >> 4] = "s>>=", -+- [BPF_END >> 4] = "endian", -+-}; -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, ",var_off=%s", tn_buf); -++ } -++ } -++ verbose(env, ")"); -++ } -++ } -++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { -++ char types_buf[BPF_REG_SIZE + 1]; -++ bool valid = false; -++ int j; -++ -++ for (j = 0; j < BPF_REG_SIZE; j++) { -++ if (state->stack[i].slot_type[j] != STACK_INVALID) -++ valid = true; -++ types_buf[j] = slot_type_char[ -++ state->stack[i].slot_type[j]]; -++ } -++ types_buf[BPF_REG_SIZE] = 0; -++ if (!valid) -++ continue; -++ verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); -++ print_liveness(env, state->stack[i].spilled_ptr.live); -++ if (state->stack[i].slot_type[0] == STACK_SPILL) { -++ reg = &state->stack[i].spilled_ptr; -++ t = reg->type; -++ verbose(env, "=%s", reg_type_str[t]); -++ if (t == SCALAR_VALUE && reg->precise) -++ verbose(env, "P"); -++ if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) -++ verbose(env, "%lld", reg->var_off.value + reg->off); -++ } else { -++ verbose(env, "=%s", types_buf); -++ } -++ } -++ if (state->acquired_refs && state->refs[0].id) { -++ verbose(env, " refs=%d", state->refs[0].id); -++ for (i = 1; i < state->acquired_refs; i++) -++ if (state->refs[i].id) -++ verbose(env, ",%d", state->refs[i].id); -++ } -++ verbose(env, "\n"); -++} -+ -+-static const char *const bpf_ldst_string[] = { -+- [BPF_W >> 3] = "u32", -+- [BPF_H >> 3] = "u16", -+- [BPF_B >> 3] = "u8", -+- [BPF_DW >> 3] = "u64", -+-}; -++#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ -++static int copy_##NAME##_state(struct bpf_func_state *dst, \ -++ const struct bpf_func_state *src) \ -++{ \ -++ if (!src->FIELD) \ -++ return 0; \ -++ if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ -++ /* internal bug, make state invalid to reject the program */ \ -++ memset(dst, 0, sizeof(*dst)); \ -++ return -EFAULT; \ -++ } \ -++ memcpy(dst->FIELD, src->FIELD, \ -++ sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ -++ return 0; \ -++} -++/* copy_reference_state() */ -++COPY_STATE_FN(reference, acquired_refs, refs, 1) -++/* copy_stack_state() */ -++COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) -++#undef COPY_STATE_FN -++ -++#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ -++static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ -++ bool copy_old) \ -++{ \ -++ u32 old_size = state->COUNT; \ -++ struct bpf_##NAME##_state *new_##FIELD; \ -++ int slot = size / SIZE; \ -++ \ -++ if (size <= old_size || !size) { \ -++ if (copy_old) \ -++ return 0; \ -++ state->COUNT = slot * SIZE; \ -++ if (!size && old_size) { \ -++ kfree(state->FIELD); \ -++ state->FIELD = NULL; \ -++ } \ -++ return 0; \ -++ } \ -++ new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ -++ GFP_KERNEL); \ -++ if (!new_##FIELD) \ -++ return -ENOMEM; \ -++ if (copy_old) { \ -++ if (state->FIELD) \ -++ memcpy(new_##FIELD, state->FIELD, \ -++ sizeof(*new_##FIELD) * (old_size / SIZE)); \ -++ memset(new_##FIELD + old_size / SIZE, 0, \ -++ sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ -++ } \ -++ state->COUNT = slot * SIZE; \ -++ kfree(state->FIELD); \ -++ state->FIELD = new_##FIELD; \ -++ return 0; \ -++} -++/* realloc_reference_state() */ -++REALLOC_STATE_FN(reference, acquired_refs, refs, 1) -++/* realloc_stack_state() */ -++REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) -++#undef REALLOC_STATE_FN -++ -++/* do_check() starts with zero-sized stack in struct bpf_verifier_state to -++ * make it consume minimal amount of memory. check_stack_write() access from -++ * the program calls into realloc_func_state() to grow the stack size. -++ * Note there is a non-zero 'parent' pointer inside bpf_verifier_state -++ * which realloc_stack_state() copies over. It points to previous -++ * bpf_verifier_state which is never reallocated. -++ */ -++static int realloc_func_state(struct bpf_func_state *state, int stack_size, -++ int refs_size, bool copy_old) -++{ -++ int err = realloc_reference_state(state, refs_size, copy_old); -++ if (err) -++ return err; -++ return realloc_stack_state(state, stack_size, copy_old); -++} -+ -+-static const char *const bpf_jmp_string[16] = { -+- [BPF_JA >> 4] = "jmp", -+- [BPF_JEQ >> 4] = "==", -+- [BPF_JGT >> 4] = ">", -+- [BPF_JGE >> 4] = ">=", -+- [BPF_JSET >> 4] = "&", -+- [BPF_JNE >> 4] = "!=", -+- [BPF_JSGT >> 4] = "s>", -+- [BPF_JSGE >> 4] = "s>=", -+- [BPF_CALL >> 4] = "call", -+- [BPF_EXIT >> 4] = "exit", -+-}; -++/* Acquire a pointer id from the env and update the state->refs to include -++ * this new pointer reference. -++ * On success, returns a valid pointer id to associate with the register -++ * On failure, returns a negative errno. -++ */ -++static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) -++{ -++ struct bpf_func_state *state = cur_func(env); -++ int new_ofs = state->acquired_refs; -++ int id, err; -+ -+-static void print_bpf_insn(const struct verifier_env *env, -+- const struct bpf_insn *insn) -++ err = realloc_reference_state(state, state->acquired_refs + 1, true); -++ if (err) -++ return err; -++ id = ++env->id_gen; -++ state->refs[new_ofs].id = id; -++ state->refs[new_ofs].insn_idx = insn_idx; -++ -++ return id; -++} -++ -++/* release function corresponding to acquire_reference_state(). Idempotent. */ -++static int release_reference_state(struct bpf_func_state *state, int ptr_id) -+ { -+- u8 class = BPF_CLASS(insn->code); -++ int i, last_idx; -+ -+- if (class == BPF_ALU || class == BPF_ALU64) { -+- if (BPF_SRC(insn->code) == BPF_X) -+- verbose("(%02x) %sr%d %s %sr%d\n", -+- insn->code, class == BPF_ALU ? "(u32) " : "", -+- insn->dst_reg, -+- bpf_alu_string[BPF_OP(insn->code) >> 4], -+- class == BPF_ALU ? "(u32) " : "", -+- insn->src_reg); -+- else -+- verbose("(%02x) %sr%d %s %s%d\n", -+- insn->code, class == BPF_ALU ? "(u32) " : "", -+- insn->dst_reg, -+- bpf_alu_string[BPF_OP(insn->code) >> 4], -+- class == BPF_ALU ? "(u32) " : "", -+- insn->imm); -+- } else if (class == BPF_STX) { -+- if (BPF_MODE(insn->code) == BPF_MEM) -+- verbose("(%02x) *(%s *)(r%d %+d) = r%d\n", -+- insn->code, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->dst_reg, -+- insn->off, insn->src_reg); -+- else if (BPF_MODE(insn->code) == BPF_XADD) -+- verbose("(%02x) lock *(%s *)(r%d %+d) += r%d\n", -+- insn->code, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->dst_reg, insn->off, -+- insn->src_reg); -+- else -+- verbose("BUG_%02x\n", insn->code); -+- } else if (class == BPF_ST) { -+- if (BPF_MODE(insn->code) != BPF_MEM) { -+- verbose("BUG_st_%02x\n", insn->code); -+- return; -+- } -+- verbose("(%02x) *(%s *)(r%d %+d) = %d\n", -+- insn->code, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->dst_reg, -+- insn->off, insn->imm); -+- } else if (class == BPF_LDX) { -+- if (BPF_MODE(insn->code) != BPF_MEM) { -+- verbose("BUG_ldx_%02x\n", insn->code); -+- return; -++ last_idx = state->acquired_refs - 1; -++ for (i = 0; i < state->acquired_refs; i++) { -++ if (state->refs[i].id == ptr_id) { -++ if (last_idx && i != last_idx) -++ memcpy(&state->refs[i], &state->refs[last_idx], -++ sizeof(*state->refs)); -++ memset(&state->refs[last_idx], 0, sizeof(*state->refs)); -++ state->acquired_refs--; -++ return 0; -+ } -+- verbose("(%02x) r%d = *(%s *)(r%d %+d)\n", -+- insn->code, insn->dst_reg, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->src_reg, insn->off); -+- } else if (class == BPF_LD) { -+- if (BPF_MODE(insn->code) == BPF_ABS) { -+- verbose("(%02x) r0 = *(%s *)skb[%d]\n", -+- insn->code, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->imm); -+- } else if (BPF_MODE(insn->code) == BPF_IND) { -+- verbose("(%02x) r0 = *(%s *)skb[r%d + %d]\n", -+- insn->code, -+- bpf_ldst_string[BPF_SIZE(insn->code) >> 3], -+- insn->src_reg, insn->imm); -+- } else if (BPF_MODE(insn->code) == BPF_IMM && -+- BPF_SIZE(insn->code) == BPF_DW) { -+- /* At this point, we already made sure that the second -+- * part of the ldimm64 insn is accessible. -+- */ -+- u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; -+- bool map_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD; -++ } -++ return -EINVAL; -++} -+ -+- if (map_ptr && !env->allow_ptr_leaks) -+- imm = 0; -++static int transfer_reference_state(struct bpf_func_state *dst, -++ struct bpf_func_state *src) -++{ -++ int err = realloc_reference_state(dst, src->acquired_refs, false); -++ if (err) -++ return err; -++ err = copy_reference_state(dst, src); -++ if (err) -++ return err; -++ return 0; -++} -+ -+- verbose("(%02x) r%d = 0x%llx\n", insn->code, -+- insn->dst_reg, (unsigned long long)imm); -+- } else { -+- verbose("BUG_ld_%02x\n", insn->code); -+- return; -+- } -+- } else if (class == BPF_JMP) { -+- u8 opcode = BPF_OP(insn->code); -++static void free_func_state(struct bpf_func_state *state) -++{ -++ if (!state) -++ return; -++ kfree(state->refs); -++ kfree(state->stack); -++ kfree(state); -++} -+ -+- if (opcode == BPF_CALL) { -+- verbose("(%02x) call %d\n", insn->code, insn->imm); -+- } else if (insn->code == (BPF_JMP | BPF_JA)) { -+- verbose("(%02x) goto pc%+d\n", -+- insn->code, insn->off); -+- } else if (insn->code == (BPF_JMP | BPF_EXIT)) { -+- verbose("(%02x) exit\n", insn->code); -+- } else if (BPF_SRC(insn->code) == BPF_X) { -+- verbose("(%02x) if r%d %s r%d goto pc%+d\n", -+- insn->code, insn->dst_reg, -+- bpf_jmp_string[BPF_OP(insn->code) >> 4], -+- insn->src_reg, insn->off); -+- } else { -+- verbose("(%02x) if r%d %s 0x%x goto pc%+d\n", -+- insn->code, insn->dst_reg, -+- bpf_jmp_string[BPF_OP(insn->code) >> 4], -+- insn->imm, insn->off); -++static void clear_jmp_history(struct bpf_verifier_state *state) -++{ -++ kfree(state->jmp_history); -++ state->jmp_history = NULL; -++ state->jmp_history_cnt = 0; -++} -++ -++static void free_verifier_state(struct bpf_verifier_state *state, -++ bool free_self) -++{ -++ int i; -++ -++ for (i = 0; i <= state->curframe; i++) { -++ free_func_state(state->frame[i]); -++ state->frame[i] = NULL; -++ } -++ clear_jmp_history(state); -++ if (free_self) -++ kfree(state); -++} -++ -++/* copy verifier state from src to dst growing dst stack space -++ * when necessary to accommodate larger src stack -++ */ -++static int copy_func_state(struct bpf_func_state *dst, -++ const struct bpf_func_state *src) -++{ -++ int err; -++ -++ err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, -++ false); -++ if (err) -++ return err; -++ memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); -++ err = copy_reference_state(dst, src); -++ if (err) -++ return err; -++ return copy_stack_state(dst, src); -++} -++ -++static int copy_verifier_state(struct bpf_verifier_state *dst_state, -++ const struct bpf_verifier_state *src) -++{ -++ struct bpf_func_state *dst; -++ u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt; -++ int i, err; -++ -++ if (dst_state->jmp_history_cnt < src->jmp_history_cnt) { -++ kfree(dst_state->jmp_history); -++ dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER); -++ if (!dst_state->jmp_history) -++ return -ENOMEM; -++ } -++ memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz); -++ dst_state->jmp_history_cnt = src->jmp_history_cnt; -++ -++ /* if dst has more stack frames then src frame, free them */ -++ for (i = src->curframe + 1; i <= dst_state->curframe; i++) { -++ free_func_state(dst_state->frame[i]); -++ dst_state->frame[i] = NULL; -++ } -++ dst_state->speculative = src->speculative; -++ dst_state->curframe = src->curframe; -++ dst_state->active_spin_lock = src->active_spin_lock; -++ dst_state->branches = src->branches; -++ dst_state->parent = src->parent; -++ dst_state->first_insn_idx = src->first_insn_idx; -++ dst_state->last_insn_idx = src->last_insn_idx; -++ for (i = 0; i <= src->curframe; i++) { -++ dst = dst_state->frame[i]; -++ if (!dst) { -++ dst = kzalloc(sizeof(*dst), GFP_KERNEL); -++ if (!dst) -++ return -ENOMEM; -++ dst_state->frame[i] = dst; -+ } -+- } else { -+- verbose("(%02x) %s\n", insn->code, bpf_class_string[class]); -++ err = copy_func_state(dst, src->frame[i]); -++ if (err) -++ return err; -+ } -++ return 0; -+ } -+ -+-static int pop_stack(struct verifier_env *env, int *prev_insn_idx) -++static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) -+ { -+- struct verifier_stack_elem *elem; -+- int insn_idx; -++ while (st) { -++ u32 br = --st->branches; -++ -++ /* WARN_ON(br > 1) technically makes sense here, -++ * but see comment in push_stack(), hence: -++ */ -++ WARN_ONCE((int)br < 0, -++ "BUG update_branch_counts:branches_to_explore=%d\n", -++ br); -++ if (br) -++ break; -++ st = st->parent; -++ } -++} -++ -++static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, -++ int *insn_idx) -++{ -++ struct bpf_verifier_state *cur = env->cur_state; -++ struct bpf_verifier_stack_elem *elem, *head = env->head; -++ int err; -+ -+ if (env->head == NULL) -+- return -1; -++ return -ENOENT; -+ -+- memcpy(&env->cur_state, &env->head->st, sizeof(env->cur_state)); -+- insn_idx = env->head->insn_idx; -++ if (cur) { -++ err = copy_verifier_state(cur, &head->st); -++ if (err) -++ return err; -++ } -++ if (insn_idx) -++ *insn_idx = head->insn_idx; -+ if (prev_insn_idx) -+- *prev_insn_idx = env->head->prev_insn_idx; -+- elem = env->head->next; -+- kfree(env->head); -++ *prev_insn_idx = head->prev_insn_idx; -++ elem = head->next; -++ free_verifier_state(&head->st, false); -++ kfree(head); -+ env->head = elem; -+ env->stack_size--; -+- return insn_idx; -++ return 0; -+ } -+ -+-static struct verifier_state *push_stack(struct verifier_env *env, int insn_idx, -+- int prev_insn_idx) -++static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, -++ int insn_idx, int prev_insn_idx, -++ bool speculative) -+ { -+- struct verifier_stack_elem *elem; -++ struct bpf_verifier_state *cur = env->cur_state; -++ struct bpf_verifier_stack_elem *elem; -++ int err; -+ -+- elem = kmalloc(sizeof(struct verifier_stack_elem), GFP_KERNEL); -++ elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); -+ if (!elem) -+ goto err; -+ -+- memcpy(&elem->st, &env->cur_state, sizeof(env->cur_state)); -+ elem->insn_idx = insn_idx; -+ elem->prev_insn_idx = prev_insn_idx; -+ elem->next = env->head; -+ env->head = elem; -+ env->stack_size++; -+- if (env->stack_size > 1024) { -+- verbose("BPF program is too complex\n"); -++ err = copy_verifier_state(&elem->st, cur); -++ if (err) -++ goto err; -++ elem->st.speculative |= speculative; -++ if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { -++ verbose(env, "The sequence of %d jumps is too complex.\n", -++ env->stack_size); -+ goto err; -+ } -++ if (elem->st.parent) { -++ ++elem->st.parent->branches; -++ /* WARN_ON(branches > 2) technically makes sense here, -++ * but -++ * 1. speculative states will bump 'branches' for non-branch -++ * instructions -++ * 2. is_state_visited() heuristics may decide not to create -++ * a new state for a sequence of branches and all such current -++ * and cloned states will be pointing to a single parent state -++ * which might have large 'branches' count. -++ */ -++ } -+ return &elem->st; -+ err: -++ free_verifier_state(env->cur_state, true); -++ env->cur_state = NULL; -+ /* pop all elements and return */ -+- while (pop_stack(env, NULL) >= 0); -++ while (!pop_stack(env, NULL, NULL)); -+ return NULL; -+ } -+ -+@@ -471,29 +852,225 @@ static const int caller_saved[CALLER_SAV -+ BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 -+ }; -+ -+-static void init_reg_state(struct reg_state *regs) -++static void __mark_reg_not_init(const struct bpf_verifier_env *env, -++ struct bpf_reg_state *reg); -++ -++/* Mark the unknown part of a register (variable offset or scalar value) as -++ * known to have the value @imm. -++ */ -++static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) -++{ -++ /* Clear id, off, and union(map_ptr, range) */ -++ memset(((u8 *)reg) + sizeof(reg->type), 0, -++ offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); -++ reg->var_off = tnum_const(imm); -++ reg->smin_value = (s64)imm; -++ reg->smax_value = (s64)imm; -++ reg->umin_value = imm; -++ reg->umax_value = imm; -++} -++ -++/* Mark the 'variable offset' part of a register as zero. This should be -++ * used only on registers holding a pointer type. -++ */ -++static void __mark_reg_known_zero(struct bpf_reg_state *reg) -++{ -++ __mark_reg_known(reg, 0); -++} -++ -++static void __mark_reg_const_zero(struct bpf_reg_state *reg) -++{ -++ __mark_reg_known(reg, 0); -++ reg->type = SCALAR_VALUE; -++} -++ -++static void mark_reg_known_zero(struct bpf_verifier_env *env, -++ struct bpf_reg_state *regs, u32 regno) -++{ -++ if (WARN_ON(regno >= MAX_BPF_REG)) { -++ verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); -++ /* Something bad happened, let's kill all regs */ -++ for (regno = 0; regno < MAX_BPF_REG; regno++) -++ __mark_reg_not_init(env, regs + regno); -++ return; -++ } -++ __mark_reg_known_zero(regs + regno); -++} -++ -++static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) -++{ -++ return type_is_pkt_pointer(reg->type); -++} -++ -++static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) -++{ -++ return reg_is_pkt_pointer(reg) || -++ reg->type == PTR_TO_PACKET_END; -++} -++ -++/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ -++static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, -++ enum bpf_reg_type which) -++{ -++ /* The register can already have a range from prior markings. -++ * This is fine as long as it hasn't been advanced from its -++ * origin. -++ */ -++ return reg->type == which && -++ reg->id == 0 && -++ reg->off == 0 && -++ tnum_equals_const(reg->var_off, 0); -++} -++ -++/* Attempts to improve min/max values based on var_off information */ -++static void __update_reg_bounds(struct bpf_reg_state *reg) -++{ -++ /* min signed is max(sign bit) | min(other bits) */ -++ reg->smin_value = max_t(s64, reg->smin_value, -++ reg->var_off.value | (reg->var_off.mask & S64_MIN)); -++ /* max signed is min(sign bit) | max(other bits) */ -++ reg->smax_value = min_t(s64, reg->smax_value, -++ reg->var_off.value | (reg->var_off.mask & S64_MAX)); -++ reg->umin_value = max(reg->umin_value, reg->var_off.value); -++ reg->umax_value = min(reg->umax_value, -++ reg->var_off.value | reg->var_off.mask); -++} -++ -++/* Uses signed min/max values to inform unsigned, and vice-versa */ -++static void __reg_deduce_bounds(struct bpf_reg_state *reg) -+ { -++ /* Learn sign from signed bounds. -++ * If we cannot cross the sign boundary, then signed and unsigned bounds -++ * are the same, so combine. This works even in the negative case, e.g. -++ * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. -++ */ -++ if (reg->smin_value >= 0 || reg->smax_value < 0) { -++ reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, -++ reg->umin_value); -++ reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, -++ reg->umax_value); -++ return; -++ } -++ /* Learn sign from unsigned bounds. Signed bounds cross the sign -++ * boundary, so we must be careful. -++ */ -++ if ((s64)reg->umax_value >= 0) { -++ /* Positive. We can't learn anything from the smin, but smax -++ * is positive, hence safe. -++ */ -++ reg->smin_value = reg->umin_value; -++ reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, -++ reg->umax_value); -++ } else if ((s64)reg->umin_value < 0) { -++ /* Negative. We can't learn anything from the smax, but smin -++ * is negative, hence safe. -++ */ -++ reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, -++ reg->umin_value); -++ reg->smax_value = reg->umax_value; -++ } -++} -++ -++/* Attempts to improve var_off based on unsigned min/max information */ -++static void __reg_bound_offset(struct bpf_reg_state *reg) -++{ -++ reg->var_off = tnum_intersect(reg->var_off, -++ tnum_range(reg->umin_value, -++ reg->umax_value)); -++} -++ -++/* Reset the min/max bounds of a register */ -++static void __mark_reg_unbounded(struct bpf_reg_state *reg) -++{ -++ reg->smin_value = S64_MIN; -++ reg->smax_value = S64_MAX; -++ reg->umin_value = 0; -++ reg->umax_value = U64_MAX; -++} -++ -++/* Mark a register as having a completely unknown (scalar) value. */ -++static void __mark_reg_unknown(const struct bpf_verifier_env *env, -++ struct bpf_reg_state *reg) -++{ -++ /* -++ * Clear type, id, off, and union(map_ptr, range) and -++ * padding between 'type' and union -++ */ -++ memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); -++ reg->type = SCALAR_VALUE; -++ reg->var_off = tnum_unknown; -++ reg->frameno = 0; -++ reg->precise = env->subprog_cnt > 1 || !env->allow_ptr_leaks ? -++ true : false; -++ __mark_reg_unbounded(reg); -++} -++ -++static void mark_reg_unknown(struct bpf_verifier_env *env, -++ struct bpf_reg_state *regs, u32 regno) -++{ -++ if (WARN_ON(regno >= MAX_BPF_REG)) { -++ verbose(env, "mark_reg_unknown(regs, %u)\n", regno); -++ /* Something bad happened, let's kill all regs except FP */ -++ for (regno = 0; regno < BPF_REG_FP; regno++) -++ __mark_reg_not_init(env, regs + regno); -++ return; -++ } -++ __mark_reg_unknown(env, regs + regno); -++} -++ -++static void __mark_reg_not_init(const struct bpf_verifier_env *env, -++ struct bpf_reg_state *reg) -++{ -++ __mark_reg_unknown(env, reg); -++ reg->type = NOT_INIT; -++} -++ -++static void mark_reg_not_init(struct bpf_verifier_env *env, -++ struct bpf_reg_state *regs, u32 regno) -++{ -++ if (WARN_ON(regno >= MAX_BPF_REG)) { -++ verbose(env, "mark_reg_not_init(regs, %u)\n", regno); -++ /* Something bad happened, let's kill all regs except FP */ -++ for (regno = 0; regno < BPF_REG_FP; regno++) -++ __mark_reg_not_init(env, regs + regno); -++ return; -++ } -++ __mark_reg_not_init(env, regs + regno); -++} -++ -++#define DEF_NOT_SUBREG (0) -++static void init_reg_state(struct bpf_verifier_env *env, -++ struct bpf_func_state *state) -++{ -++ struct bpf_reg_state *regs = state->regs; -+ int i; -+ -+ for (i = 0; i < MAX_BPF_REG; i++) { -+- regs[i].type = NOT_INIT; -+- regs[i].imm = 0; -+- regs[i].map_ptr = NULL; -++ mark_reg_not_init(env, regs, i); -++ regs[i].live = REG_LIVE_NONE; -++ regs[i].parent = NULL; -++ regs[i].subreg_def = DEF_NOT_SUBREG; -+ } -+ -+ /* frame pointer */ -+- regs[BPF_REG_FP].type = FRAME_PTR; -++ regs[BPF_REG_FP].type = PTR_TO_STACK; -++ mark_reg_known_zero(env, regs, BPF_REG_FP); -++ regs[BPF_REG_FP].frameno = state->frameno; -+ -+ /* 1st arg to a function */ -+ regs[BPF_REG_1].type = PTR_TO_CTX; -++ mark_reg_known_zero(env, regs, BPF_REG_1); -+ } -+ -+-static void mark_reg_unknown_value(struct reg_state *regs, u32 regno) -++#define BPF_MAIN_FUNC (-1) -++static void init_func_state(struct bpf_verifier_env *env, -++ struct bpf_func_state *state, -++ int callsite, int frameno, int subprogno) -+ { -+- BUG_ON(regno >= MAX_BPF_REG); -+- regs[regno].type = UNKNOWN_VALUE; -+- regs[regno].imm = 0; -+- regs[regno].map_ptr = NULL; -++ state->callsite = callsite; -++ state->frameno = frameno; -++ state->subprogno = subprogno; -++ init_reg_state(env, state); -+ } -+ -+ enum reg_arg_type { -+@@ -502,44 +1079,760 @@ enum reg_arg_type { -+ DST_OP_NO_MARK /* same as above, check only, don't mark */ -+ }; -+ -+-static int check_reg_arg(struct reg_state *regs, u32 regno, -++static int cmp_subprogs(const void *a, const void *b) -++{ -++ return ((struct bpf_subprog_info *)a)->start - -++ ((struct bpf_subprog_info *)b)->start; -++} -++ -++static int find_subprog(struct bpf_verifier_env *env, int off) -++{ -++ struct bpf_subprog_info *p; -++ -++ p = bsearch(&off, env->subprog_info, env->subprog_cnt, -++ sizeof(env->subprog_info[0]), cmp_subprogs); -++ if (!p) -++ return -ENOENT; -++ return p - env->subprog_info; -++ -++} -++ -++static int add_subprog(struct bpf_verifier_env *env, int off) -++{ -++ int insn_cnt = env->prog->len; -++ int ret; -++ -++ if (off >= insn_cnt || off < 0) { -++ verbose(env, "call to invalid destination\n"); -++ return -EINVAL; -++ } -++ ret = find_subprog(env, off); -++ if (ret >= 0) -++ return 0; -++ if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { -++ verbose(env, "too many subprograms\n"); -++ return -E2BIG; -++ } -++ env->subprog_info[env->subprog_cnt++].start = off; -++ sort(env->subprog_info, env->subprog_cnt, -++ sizeof(env->subprog_info[0]), cmp_subprogs, NULL); -++ return 0; -++} -++ -++static int check_subprogs(struct bpf_verifier_env *env) -++{ -++ int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; -++ struct bpf_subprog_info *subprog = env->subprog_info; -++ struct bpf_insn *insn = env->prog->insnsi; -++ int insn_cnt = env->prog->len; -++ -++ /* Add entry function. */ -++ ret = add_subprog(env, 0); -++ if (ret < 0) -++ return ret; -++ -++ /* determine subprog starts. The end is one before the next starts */ -++ for (i = 0; i < insn_cnt; i++) { -++ if (insn[i].code != (BPF_JMP | BPF_CALL)) -++ continue; -++ if (insn[i].src_reg != BPF_PSEUDO_CALL) -++ continue; -++ if (!env->allow_ptr_leaks) { -++ verbose(env, "function calls to other bpf functions are allowed for root only\n"); -++ return -EPERM; -++ } -++ ret = add_subprog(env, i + insn[i].imm + 1); -++ if (ret < 0) -++ return ret; -++ } -++ -++ /* Add a fake 'exit' subprog which could simplify subprog iteration -++ * logic. 'subprog_cnt' should not be increased. -++ */ -++ subprog[env->subprog_cnt].start = insn_cnt; -++ -++ if (env->log.level & BPF_LOG_LEVEL2) -++ for (i = 0; i < env->subprog_cnt; i++) -++ verbose(env, "func#%d @%d\n", i, subprog[i].start); -++ -++ /* now check that all jumps are within the same subprog */ -++ subprog_start = subprog[cur_subprog].start; -++ subprog_end = subprog[cur_subprog + 1].start; -++ for (i = 0; i < insn_cnt; i++) { -++ u8 code = insn[i].code; -++ -++ if (code == (BPF_JMP | BPF_CALL) && -++ insn[i].imm == BPF_FUNC_tail_call && -++ insn[i].src_reg != BPF_PSEUDO_CALL) -++ subprog[cur_subprog].has_tail_call = true; -++ if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) -++ goto next; -++ if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) -++ goto next; -++ off = i + insn[i].off + 1; -++ if (off < subprog_start || off >= subprog_end) { -++ verbose(env, "jump out of range from insn %d to %d\n", i, off); -++ return -EINVAL; -++ } -++next: -++ if (i == subprog_end - 1) { -++ /* to avoid fall-through from one subprog into another -++ * the last insn of the subprog should be either exit -++ * or unconditional jump back -++ */ -++ if (code != (BPF_JMP | BPF_EXIT) && -++ code != (BPF_JMP | BPF_JA)) { -++ verbose(env, "last insn is not an exit or jmp\n"); -++ return -EINVAL; -++ } -++ subprog_start = subprog_end; -++ cur_subprog++; -++ if (cur_subprog < env->subprog_cnt) -++ subprog_end = subprog[cur_subprog + 1].start; -++ } -++ } -++ return 0; -++} -++ -++/* Parentage chain of this register (or stack slot) should take care of all -++ * issues like callee-saved registers, stack slot allocation time, etc. -++ */ -++static int mark_reg_read(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *state, -++ struct bpf_reg_state *parent, u8 flag) -++{ -++ bool writes = parent == state->parent; /* Observe write marks */ -++ int cnt = 0; -++ -++ while (parent) { -++ /* if read wasn't screened by an earlier write ... */ -++ if (writes && state->live & REG_LIVE_WRITTEN) -++ break; -++ if (parent->live & REG_LIVE_DONE) { -++ verbose(env, "verifier BUG type %s var_off %lld off %d\n", -++ reg_type_str[parent->type], -++ parent->var_off.value, parent->off); -++ return -EFAULT; -++ } -++ /* The first condition is more likely to be true than the -++ * second, checked it first. -++ */ -++ if ((parent->live & REG_LIVE_READ) == flag || -++ parent->live & REG_LIVE_READ64) -++ /* The parentage chain never changes and -++ * this parent was already marked as LIVE_READ. -++ * There is no need to keep walking the chain again and -++ * keep re-marking all parents as LIVE_READ. -++ * This case happens when the same register is read -++ * multiple times without writes into it in-between. -++ * Also, if parent has the stronger REG_LIVE_READ64 set, -++ * then no need to set the weak REG_LIVE_READ32. -++ */ -++ break; -++ /* ... then we depend on parent's value */ -++ parent->live |= flag; -++ /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ -++ if (flag == REG_LIVE_READ64) -++ parent->live &= ~REG_LIVE_READ32; -++ state = parent; -++ parent = state->parent; -++ writes = true; -++ cnt++; -++ } -++ -++ if (env->longest_mark_read_walk < cnt) -++ env->longest_mark_read_walk = cnt; -++ return 0; -++} -++ -++/* This function is supposed to be used by the following 32-bit optimization -++ * code only. It returns TRUE if the source or destination register operates -++ * on 64-bit, otherwise return FALSE. -++ */ -++static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, -++ u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) -++{ -++ u8 code, class, op; -++ -++ code = insn->code; -++ class = BPF_CLASS(code); -++ op = BPF_OP(code); -++ if (class == BPF_JMP) { -++ /* BPF_EXIT for "main" will reach here. Return TRUE -++ * conservatively. -++ */ -++ if (op == BPF_EXIT) -++ return true; -++ if (op == BPF_CALL) { -++ /* BPF to BPF call will reach here because of marking -++ * caller saved clobber with DST_OP_NO_MARK for which we -++ * don't care the register def because they are anyway -++ * marked as NOT_INIT already. -++ */ -++ if (insn->src_reg == BPF_PSEUDO_CALL) -++ return false; -++ /* Helper call will reach here because of arg type -++ * check, conservatively return TRUE. -++ */ -++ if (t == SRC_OP) -++ return true; -++ -++ return false; -++ } -++ } -++ -++ if (class == BPF_ALU64 || class == BPF_JMP || -++ /* BPF_END always use BPF_ALU class. */ -++ (class == BPF_ALU && op == BPF_END && insn->imm == 64)) -++ return true; -++ -++ if (class == BPF_ALU || class == BPF_JMP32) -++ return false; -++ -++ if (class == BPF_LDX) { -++ if (t != SRC_OP) -++ return BPF_SIZE(code) == BPF_DW; -++ /* LDX source must be ptr. */ -++ return true; -++ } -++ -++ if (class == BPF_STX) { -++ if (reg->type != SCALAR_VALUE) -++ return true; -++ return BPF_SIZE(code) == BPF_DW; -++ } -++ -++ if (class == BPF_LD) { -++ u8 mode = BPF_MODE(code); -++ -++ /* LD_IMM64 */ -++ if (mode == BPF_IMM) -++ return true; -++ -++ /* Both LD_IND and LD_ABS return 32-bit data. */ -++ if (t != SRC_OP) -++ return false; -++ -++ /* Implicit ctx ptr. */ -++ if (regno == BPF_REG_6) -++ return true; -++ -++ /* Explicit source could be any width. */ -++ return true; -++ } -++ -++ if (class == BPF_ST) -++ /* The only source register for BPF_ST is a ptr. */ -++ return true; -++ -++ /* Conservatively return true at default. */ -++ return true; -++} -++ -++/* Return TRUE if INSN doesn't have explicit value define. */ -++static bool insn_no_def(struct bpf_insn *insn) -++{ -++ u8 class = BPF_CLASS(insn->code); -++ -++ return (class == BPF_JMP || class == BPF_JMP32 || -++ class == BPF_STX || class == BPF_ST); -++} -++ -++/* Return TRUE if INSN has defined any 32-bit value explicitly. */ -++static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) -++{ -++ if (insn_no_def(insn)) -++ return false; -++ -++ return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP); -++} -++ -++static void mark_insn_zext(struct bpf_verifier_env *env, -++ struct bpf_reg_state *reg) -++{ -++ s32 def_idx = reg->subreg_def; -++ -++ if (def_idx == DEF_NOT_SUBREG) -++ return; -++ -++ env->insn_aux_data[def_idx - 1].zext_dst = true; -++ /* The dst will be zero extended, so won't be sub-register anymore. */ -++ reg->subreg_def = DEF_NOT_SUBREG; -++} -++ -++static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, -+ enum reg_arg_type t) -+ { -++ struct bpf_verifier_state *vstate = env->cur_state; -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; -++ struct bpf_reg_state *reg, *regs = state->regs; -++ bool rw64; -++ -+ if (regno >= MAX_BPF_REG) { -+- verbose("R%d is invalid\n", regno); -++ verbose(env, "R%d is invalid\n", regno); -+ return -EINVAL; -+ } -+ -++ reg = ®s[regno]; -++ rw64 = is_reg64(env, insn, regno, reg, t); -+ if (t == SRC_OP) { -+ /* check whether register used as source operand can be read */ -+- if (regs[regno].type == NOT_INIT) { -+- verbose("R%d !read_ok\n", regno); -++ if (reg->type == NOT_INIT) { -++ verbose(env, "R%d !read_ok\n", regno); -+ return -EACCES; -+ } -++ /* We don't need to worry about FP liveness because it's read-only */ -++ if (regno == BPF_REG_FP) -++ return 0; -++ -++ if (rw64) -++ mark_insn_zext(env, reg); -++ -++ return mark_reg_read(env, reg, reg->parent, -++ rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); -+ } else { -+ /* check whether register used as dest operand can be written to */ -+ if (regno == BPF_REG_FP) { -+- verbose("frame pointer is read only\n"); -++ verbose(env, "frame pointer is read only\n"); -+ return -EACCES; -+ } -++ reg->live |= REG_LIVE_WRITTEN; -++ reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; -+ if (t == DST_OP) -+- mark_reg_unknown_value(regs, regno); -++ mark_reg_unknown(env, regs, regno); -+ } -+ return 0; -+ } -+ -+-static int bpf_size_to_bytes(int bpf_size) -++/* for any branch, call, exit record the history of jmps in the given state */ -++static int push_jmp_history(struct bpf_verifier_env *env, -++ struct bpf_verifier_state *cur) -+ { -+- if (bpf_size == BPF_W) -+- return 4; -+- else if (bpf_size == BPF_H) -+- return 2; -+- else if (bpf_size == BPF_B) -+- return 1; -+- else if (bpf_size == BPF_DW) -+- return 8; -+- else -+- return -EINVAL; -++ u32 cnt = cur->jmp_history_cnt; -++ struct bpf_idx_pair *p; -++ -++ cnt++; -++ p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); -++ if (!p) -++ return -ENOMEM; -++ p[cnt - 1].idx = env->insn_idx; -++ p[cnt - 1].prev_idx = env->prev_insn_idx; -++ cur->jmp_history = p; -++ cur->jmp_history_cnt = cnt; -++ return 0; -++} -++ -++/* Backtrack one insn at a time. If idx is not at the top of recorded -++ * history then previous instruction came from straight line execution. -++ */ -++static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, -++ u32 *history) -++{ -++ u32 cnt = *history; -++ -++ if (cnt && st->jmp_history[cnt - 1].idx == i) { -++ i = st->jmp_history[cnt - 1].prev_idx; -++ (*history)--; -++ } else { -++ i--; -++ } -++ return i; -++} -++ -++/* For given verifier state backtrack_insn() is called from the last insn to -++ * the first insn. Its purpose is to compute a bitmask of registers and -++ * stack slots that needs precision in the parent verifier state. -++ */ -++static int backtrack_insn(struct bpf_verifier_env *env, int idx, -++ u32 *reg_mask, u64 *stack_mask) -++{ -++ const struct bpf_insn_cbs cbs = { -++ .cb_print = verbose, -++ .private_data = env, -++ }; -++ struct bpf_insn *insn = env->prog->insnsi + idx; -++ u8 class = BPF_CLASS(insn->code); -++ u8 opcode = BPF_OP(insn->code); -++ u8 mode = BPF_MODE(insn->code); -++ u32 dreg = 1u << insn->dst_reg; -++ u32 sreg = 1u << insn->src_reg; -++ u32 spi; -++ -++ if (insn->code == 0) -++ return 0; -++ if (env->log.level & BPF_LOG_LEVEL) { -++ verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); -++ verbose(env, "%d: ", idx); -++ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); -++ } -++ -++ if (class == BPF_ALU || class == BPF_ALU64) { -++ if (!(*reg_mask & dreg)) -++ return 0; -++ if (opcode == BPF_MOV) { -++ if (BPF_SRC(insn->code) == BPF_X) { -++ /* dreg = sreg -++ * dreg needs precision after this insn -++ * sreg needs precision before this insn -++ */ -++ *reg_mask &= ~dreg; -++ *reg_mask |= sreg; -++ } else { -++ /* dreg = K -++ * dreg needs precision after this insn. -++ * Corresponding register is already marked -++ * as precise=true in this verifier state. -++ * No further markings in parent are necessary -++ */ -++ *reg_mask &= ~dreg; -++ } -++ } else { -++ if (BPF_SRC(insn->code) == BPF_X) { -++ /* dreg += sreg -++ * both dreg and sreg need precision -++ * before this insn -++ */ -++ *reg_mask |= sreg; -++ } /* else dreg += K -++ * dreg still needs precision before this insn -++ */ -++ } -++ } else if (class == BPF_LDX) { -++ if (!(*reg_mask & dreg)) -++ return 0; -++ *reg_mask &= ~dreg; -++ -++ /* scalars can only be spilled into stack w/o losing precision. -++ * Load from any other memory can be zero extended. -++ * The desire to keep that precision is already indicated -++ * by 'precise' mark in corresponding register of this state. -++ * No further tracking necessary. -++ */ -++ if (insn->src_reg != BPF_REG_FP) -++ return 0; -++ if (BPF_SIZE(insn->code) != BPF_DW) -++ return 0; -++ -++ /* dreg = *(u64 *)[fp - off] was a fill from the stack. -++ * that [fp - off] slot contains scalar that needs to be -++ * tracked with precision -++ */ -++ spi = (-insn->off - 1) / BPF_REG_SIZE; -++ if (spi >= 64) { -++ verbose(env, "BUG spi %d\n", spi); -++ WARN_ONCE(1, "verifier backtracking bug"); -++ return -EFAULT; -++ } -++ *stack_mask |= 1ull << spi; -++ } else if (class == BPF_STX || class == BPF_ST) { -++ if (*reg_mask & dreg) -++ /* stx & st shouldn't be using _scalar_ dst_reg -++ * to access memory. It means backtracking -++ * encountered a case of pointer subtraction. -++ */ -++ return -ENOTSUPP; -++ /* scalars can only be spilled into stack */ -++ if (insn->dst_reg != BPF_REG_FP) -++ return 0; -++ if (BPF_SIZE(insn->code) != BPF_DW) -++ return 0; -++ spi = (-insn->off - 1) / BPF_REG_SIZE; -++ if (spi >= 64) { -++ verbose(env, "BUG spi %d\n", spi); -++ WARN_ONCE(1, "verifier backtracking bug"); -++ return -EFAULT; -++ } -++ if (!(*stack_mask & (1ull << spi))) -++ return 0; -++ *stack_mask &= ~(1ull << spi); -++ if (class == BPF_STX) -++ *reg_mask |= sreg; -++ } else if (class == BPF_JMP || class == BPF_JMP32) { -++ if (opcode == BPF_CALL) { -++ if (insn->src_reg == BPF_PSEUDO_CALL) -++ return -ENOTSUPP; -++ /* regular helper call sets R0 */ -++ *reg_mask &= ~1; -++ if (*reg_mask & 0x3f) { -++ /* if backtracing was looking for registers R1-R5 -++ * they should have been found already. -++ */ -++ verbose(env, "BUG regs %x\n", *reg_mask); -++ WARN_ONCE(1, "verifier backtracking bug"); -++ return -EFAULT; -++ } -++ } else if (opcode == BPF_EXIT) { -++ return -ENOTSUPP; -++ } -++ } else if (class == BPF_LD) { -++ if (!(*reg_mask & dreg)) -++ return 0; -++ *reg_mask &= ~dreg; -++ /* It's ld_imm64 or ld_abs or ld_ind. -++ * For ld_imm64 no further tracking of precision -++ * into parent is necessary -++ */ -++ if (mode == BPF_IND || mode == BPF_ABS) -++ /* to be analyzed */ -++ return -ENOTSUPP; -++ } -++ return 0; -++} -++ -++/* the scalar precision tracking algorithm: -++ * . at the start all registers have precise=false. -++ * . scalar ranges are tracked as normal through alu and jmp insns. -++ * . once precise value of the scalar register is used in: -++ * . ptr + scalar alu -++ * . if (scalar cond K|scalar) -++ * . helper_call(.., scalar, ...) where ARG_CONST is expected -++ * backtrack through the verifier states and mark all registers and -++ * stack slots with spilled constants that these scalar regisers -++ * should be precise. -++ * . during state pruning two registers (or spilled stack slots) -++ * are equivalent if both are not precise. -++ * -++ * Note the verifier cannot simply walk register parentage chain, -++ * since many different registers and stack slots could have been -++ * used to compute single precise scalar. -++ * -++ * The approach of starting with precise=true for all registers and then -++ * backtrack to mark a register as not precise when the verifier detects -++ * that program doesn't care about specific value (e.g., when helper -++ * takes register as ARG_ANYTHING parameter) is not safe. -++ * -++ * It's ok to walk single parentage chain of the verifier states. -++ * It's possible that this backtracking will go all the way till 1st insn. -++ * All other branches will be explored for needing precision later. -++ * -++ * The backtracking needs to deal with cases like: -++ * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) -++ * r9 -= r8 -++ * r5 = r9 -++ * if r5 > 0x79f goto pc+7 -++ * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) -++ * r5 += 1 -++ * ... -++ * call bpf_perf_event_output#25 -++ * where .arg5_type = ARG_CONST_SIZE_OR_ZERO -++ * -++ * and this case: -++ * r6 = 1 -++ * call foo // uses callee's r6 inside to compute r0 -++ * r0 += r6 -++ * if r0 == 0 goto -++ * -++ * to track above reg_mask/stack_mask needs to be independent for each frame. -++ * -++ * Also if parent's curframe > frame where backtracking started, -++ * the verifier need to mark registers in both frames, otherwise callees -++ * may incorrectly prune callers. This is similar to -++ * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") -++ * -++ * For now backtracking falls back into conservative marking. -++ */ -++static void mark_all_scalars_precise(struct bpf_verifier_env *env, -++ struct bpf_verifier_state *st) -++{ -++ struct bpf_func_state *func; -++ struct bpf_reg_state *reg; -++ int i, j; -++ -++ /* big hammer: mark all scalars precise in this path. -++ * pop_stack may still get !precise scalars. -++ */ -++ for (; st; st = st->parent) -++ for (i = 0; i <= st->curframe; i++) { -++ func = st->frame[i]; -++ for (j = 0; j < BPF_REG_FP; j++) { -++ reg = &func->regs[j]; -++ if (reg->type != SCALAR_VALUE) -++ continue; -++ reg->precise = true; -++ } -++ for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { -++ if (func->stack[j].slot_type[0] != STACK_SPILL) -++ continue; -++ reg = &func->stack[j].spilled_ptr; -++ if (reg->type != SCALAR_VALUE) -++ continue; -++ reg->precise = true; -++ } -++ } -++} -++ -++static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, -++ int spi) -++{ -++ struct bpf_verifier_state *st = env->cur_state; -++ int first_idx = st->first_insn_idx; -++ int last_idx = env->insn_idx; -++ struct bpf_func_state *func; -++ struct bpf_reg_state *reg; -++ u32 reg_mask = regno >= 0 ? 1u << regno : 0; -++ u64 stack_mask = spi >= 0 ? 1ull << spi : 0; -++ bool skip_first = true; -++ bool new_marks = false; -++ int i, err; -++ -++ if (!env->allow_ptr_leaks) -++ /* backtracking is root only for now */ -++ return 0; -++ -++ func = st->frame[st->curframe]; -++ if (regno >= 0) { -++ reg = &func->regs[regno]; -++ if (reg->type != SCALAR_VALUE) { -++ WARN_ONCE(1, "backtracing misuse"); -++ return -EFAULT; -++ } -++ if (!reg->precise) -++ new_marks = true; -++ else -++ reg_mask = 0; -++ reg->precise = true; -++ } -++ -++ while (spi >= 0) { -++ if (func->stack[spi].slot_type[0] != STACK_SPILL) { -++ stack_mask = 0; -++ break; -++ } -++ reg = &func->stack[spi].spilled_ptr; -++ if (reg->type != SCALAR_VALUE) { -++ stack_mask = 0; -++ break; -++ } -++ if (!reg->precise) -++ new_marks = true; -++ else -++ stack_mask = 0; -++ reg->precise = true; -++ break; -++ } -++ -++ if (!new_marks) -++ return 0; -++ if (!reg_mask && !stack_mask) -++ return 0; -++ for (;;) { -++ DECLARE_BITMAP(mask, 64); -++ u32 history = st->jmp_history_cnt; -++ -++ if (env->log.level & BPF_LOG_LEVEL) -++ verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); -++ for (i = last_idx;;) { -++ if (skip_first) { -++ err = 0; -++ skip_first = false; -++ } else { -++ err = backtrack_insn(env, i, ®_mask, &stack_mask); -++ } -++ if (err == -ENOTSUPP) { -++ mark_all_scalars_precise(env, st); -++ return 0; -++ } else if (err) { -++ return err; -++ } -++ if (!reg_mask && !stack_mask) -++ /* Found assignment(s) into tracked register in this state. -++ * Since this state is already marked, just return. -++ * Nothing to be tracked further in the parent state. -++ */ -++ return 0; -++ if (i == first_idx) -++ break; -++ i = get_prev_insn_idx(st, i, &history); -++ if (i >= env->prog->len) { -++ /* This can happen if backtracking reached insn 0 -++ * and there are still reg_mask or stack_mask -++ * to backtrack. -++ * It means the backtracking missed the spot where -++ * particular register was initialized with a constant. -++ */ -++ verbose(env, "BUG backtracking idx %d\n", i); -++ WARN_ONCE(1, "verifier backtracking bug"); -++ return -EFAULT; -++ } -++ } -++ st = st->parent; -++ if (!st) -++ break; -++ -++ new_marks = false; -++ func = st->frame[st->curframe]; -++ bitmap_from_u64(mask, reg_mask); -++ for_each_set_bit(i, mask, 32) { -++ reg = &func->regs[i]; -++ if (reg->type != SCALAR_VALUE) { -++ reg_mask &= ~(1u << i); -++ continue; -++ } -++ if (!reg->precise) -++ new_marks = true; -++ reg->precise = true; -++ } -++ -++ bitmap_from_u64(mask, stack_mask); -++ for_each_set_bit(i, mask, 64) { -++ if (i >= func->allocated_stack / BPF_REG_SIZE) { -++ /* the sequence of instructions: -++ * 2: (bf) r3 = r10 -++ * 3: (7b) *(u64 *)(r3 -8) = r0 -++ * 4: (79) r4 = *(u64 *)(r10 -8) -++ * doesn't contain jmps. It's backtracked -++ * as a single block. -++ * During backtracking insn 3 is not recognized as -++ * stack access, so at the end of backtracking -++ * stack slot fp-8 is still marked in stack_mask. -++ * However the parent state may not have accessed -++ * fp-8 and it's "unallocated" stack space. -++ * In such case fallback to conservative. -++ */ -++ mark_all_scalars_precise(env, st); -++ return 0; -++ } -++ -++ if (func->stack[i].slot_type[0] != STACK_SPILL) { -++ stack_mask &= ~(1ull << i); -++ continue; -++ } -++ reg = &func->stack[i].spilled_ptr; -++ if (reg->type != SCALAR_VALUE) { -++ stack_mask &= ~(1ull << i); -++ continue; -++ } -++ if (!reg->precise) -++ new_marks = true; -++ reg->precise = true; -++ } -++ if (env->log.level & BPF_LOG_LEVEL) { -++ print_verifier_state(env, func); -++ verbose(env, "parent %s regs=%x stack=%llx marks\n", -++ new_marks ? "didn't have" : "already had", -++ reg_mask, stack_mask); -++ } -++ -++ if (!reg_mask && !stack_mask) -++ break; -++ if (!new_marks) -++ break; -++ -++ last_idx = st->last_insn_idx; -++ first_idx = st->first_insn_idx; -++ } -++ return 0; -++} -++ -++static int mark_chain_precision(struct bpf_verifier_env *env, int regno) -++{ -++ return __mark_chain_precision(env, regno, -1); -++} -++ -++static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) -++{ -++ return __mark_chain_precision(env, -1, spi); -+ } -+ -+ static bool is_spillable_regtype(enum bpf_reg_type type) -+@@ -549,129 +1842,932 @@ static bool is_spillable_regtype(enum bp -+ case PTR_TO_MAP_VALUE_OR_NULL: -+ case PTR_TO_STACK: -+ case PTR_TO_CTX: -+- case FRAME_PTR: -++ case PTR_TO_PACKET: -++ case PTR_TO_PACKET_META: -++ case PTR_TO_PACKET_END: -++ case PTR_TO_FLOW_KEYS: -+ case CONST_PTR_TO_MAP: -++ case PTR_TO_SOCKET: -++ case PTR_TO_SOCKET_OR_NULL: -++ case PTR_TO_SOCK_COMMON: -++ case PTR_TO_SOCK_COMMON_OR_NULL: -++ case PTR_TO_TCP_SOCK: -++ case PTR_TO_TCP_SOCK_OR_NULL: -++ case PTR_TO_XDP_SOCK: -+ return true; -+ default: -+ return false; -+ } -+ } -+ -++/* Does this register contain a constant zero? */ -++static bool register_is_null(struct bpf_reg_state *reg) -++{ -++ return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); -++} -++ -++static bool register_is_const(struct bpf_reg_state *reg) -++{ -++ return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); -++} -++ -++static bool __is_pointer_value(bool allow_ptr_leaks, -++ const struct bpf_reg_state *reg) -++{ -++ if (allow_ptr_leaks) -++ return false; -++ -++ return reg->type != SCALAR_VALUE; -++} -++ -++static void save_register_state(struct bpf_func_state *state, -++ int spi, struct bpf_reg_state *reg) -++{ -++ int i; -++ -++ state->stack[spi].spilled_ptr = *reg; -++ state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; -++ -++ for (i = 0; i < BPF_REG_SIZE; i++) -++ state->stack[spi].slot_type[i] = STACK_SPILL; -++} -++ -+ /* check_stack_read/write functions track spill/fill of registers, -+ * stack boundary and alignment are checked in check_mem_access() -+ */ -+-static int check_stack_write(struct verifier_state *state, int off, int size, -+- int value_regno) -++static int check_stack_write(struct bpf_verifier_env *env, -++ struct bpf_func_state *state, /* func where register points to */ -++ int off, int size, int value_regno, int insn_idx) -+ { -+- int i; -++ struct bpf_func_state *cur; /* state of the current function */ -++ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; -++ u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; -++ struct bpf_reg_state *reg = NULL; -++ -++ err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), -++ state->acquired_refs, true); -++ if (err) -++ return err; -+ /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, -+ * so it's aligned access and [off, off + size) are within stack limits -+ */ -++ if (!env->allow_ptr_leaks && -++ state->stack[spi].slot_type[0] == STACK_SPILL && -++ size != BPF_REG_SIZE) { -++ verbose(env, "attempt to corrupt spilled pointer on stack\n"); -++ return -EACCES; -++ } -+ -+- if (value_regno >= 0 && -+- is_spillable_regtype(state->regs[value_regno].type)) { -+- -++ cur = env->cur_state->frame[env->cur_state->curframe]; -++ if (value_regno >= 0) -++ reg = &cur->regs[value_regno]; -++ -++ if (reg && size == BPF_REG_SIZE && register_is_const(reg) && -++ !register_is_null(reg) && env->allow_ptr_leaks) { -++ if (dst_reg != BPF_REG_FP) { -++ /* The backtracking logic can only recognize explicit -++ * stack slot address like [fp - 8]. Other spill of -++ * scalar via different register has to be conervative. -++ * Backtrack from here and mark all registers as precise -++ * that contributed into 'reg' being a constant. -++ */ -++ err = mark_chain_precision(env, value_regno); -++ if (err) -++ return err; -++ } -++ save_register_state(state, spi, reg); -++ } else if (reg && is_spillable_regtype(reg->type)) { -+ /* register containing pointer is being spilled into stack */ -+ if (size != BPF_REG_SIZE) { -+- verbose("invalid size of register spill\n"); -++ verbose_linfo(env, insn_idx, "; "); -++ verbose(env, "invalid size of register spill\n"); -+ return -EACCES; -+ } -+ -+- /* save register state */ -+- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE] = -+- state->regs[value_regno]; -++ if (state != cur && reg->type == PTR_TO_STACK) { -++ verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); -++ return -EINVAL; -++ } -+ -+- for (i = 0; i < BPF_REG_SIZE; i++) -+- state->stack_slot_type[MAX_BPF_STACK + off + i] = STACK_SPILL; -++ if (!env->allow_ptr_leaks) { -++ bool sanitize = false; -++ -++ if (state->stack[spi].slot_type[0] == STACK_SPILL && -++ register_is_const(&state->stack[spi].spilled_ptr)) -++ sanitize = true; -++ for (i = 0; i < BPF_REG_SIZE; i++) -++ if (state->stack[spi].slot_type[i] == STACK_MISC) { -++ sanitize = true; -++ break; -++ } -++ if (sanitize) { -++ int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; -++ int soff = (-spi - 1) * BPF_REG_SIZE; -++ -++ /* detected reuse of integer stack slot with a pointer -++ * which means either llvm is reusing stack slot or -++ * an attacker is trying to exploit CVE-2018-3639 -++ * (speculative store bypass) -++ * Have to sanitize that slot with preemptive -++ * store of zero. -++ */ -++ if (*poff && *poff != soff) { -++ /* disallow programs where single insn stores -++ * into two different stack slots, since verifier -++ * cannot sanitize them -++ */ -++ verbose(env, -++ "insn %d cannot access two stack slots fp%d and fp%d", -++ insn_idx, *poff, soff); -++ return -EINVAL; -++ } -++ *poff = soff; -++ } -++ } -++ save_register_state(state, spi, reg); -+ } else { -+- /* regular write of data into stack */ -+- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE] = -+- (struct reg_state) {}; -++ u8 type = STACK_MISC; -++ -++ /* regular write of data into stack destroys any spilled ptr */ -++ state->stack[spi].spilled_ptr.type = NOT_INIT; -++ /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ -++ if (state->stack[spi].slot_type[0] == STACK_SPILL) -++ for (i = 0; i < BPF_REG_SIZE; i++) -++ state->stack[spi].slot_type[i] = STACK_MISC; -++ -++ /* only mark the slot as written if all 8 bytes were written -++ * otherwise read propagation may incorrectly stop too soon -++ * when stack slots are partially written. -++ * This heuristic means that read propagation will be -++ * conservative, since it will add reg_live_read marks -++ * to stack slots all the way to first state when programs -++ * writes+reads less than 8 bytes -++ */ -++ if (size == BPF_REG_SIZE) -++ state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; -++ -++ /* when we zero initialize stack slots mark them as such */ -++ if (reg && register_is_null(reg)) { -++ /* backtracking doesn't work for STACK_ZERO yet. */ -++ err = mark_chain_precision(env, value_regno); -++ if (err) -++ return err; -++ type = STACK_ZERO; -++ } -+ -++ /* Mark slots affected by this stack write. */ -+ for (i = 0; i < size; i++) -+- state->stack_slot_type[MAX_BPF_STACK + off + i] = STACK_MISC; -++ state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = -++ type; -+ } -+ return 0; -+ } -+ -+-static int check_stack_read(struct verifier_state *state, int off, int size, -+- int value_regno) -++static int check_stack_read(struct bpf_verifier_env *env, -++ struct bpf_func_state *reg_state /* func where register points to */, -++ int off, int size, int value_regno) -+ { -+- u8 *slot_type; -+- int i; -+- -+- slot_type = &state->stack_slot_type[MAX_BPF_STACK + off]; -++ struct bpf_verifier_state *vstate = env->cur_state; -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; -++ struct bpf_reg_state *reg; -++ u8 *stype; -++ -++ if (reg_state->allocated_stack <= slot) { -++ verbose(env, "invalid read from stack off %d+0 size %d\n", -++ off, size); -++ return -EACCES; -++ } -++ stype = reg_state->stack[spi].slot_type; -++ reg = ®_state->stack[spi].spilled_ptr; -+ -+- if (slot_type[0] == STACK_SPILL) { -++ if (stype[0] == STACK_SPILL) { -+ if (size != BPF_REG_SIZE) { -+- verbose("invalid size of register spill\n"); -+- return -EACCES; -++ if (reg->type != SCALAR_VALUE) { -++ verbose_linfo(env, env->insn_idx, "; "); -++ verbose(env, "invalid size of register fill\n"); -++ return -EACCES; -++ } -++ if (value_regno >= 0) { -++ mark_reg_unknown(env, state->regs, value_regno); -++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; -++ } -++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); -++ return 0; -+ } -+ for (i = 1; i < BPF_REG_SIZE; i++) { -+- if (slot_type[i] != STACK_SPILL) { -+- verbose("corrupted spill memory\n"); -++ if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { -++ verbose(env, "corrupted spill memory\n"); -+ return -EACCES; -+ } -+ } -+ -+- if (value_regno >= 0) -++ if (value_regno >= 0) { -+ /* restore register state from stack */ -+- state->regs[value_regno] = -+- state->spilled_regs[(MAX_BPF_STACK + off) / BPF_REG_SIZE]; -+- return 0; -++ state->regs[value_regno] = *reg; -++ /* mark reg as written since spilled pointer state likely -++ * has its liveness marks cleared by is_state_visited() -++ * which resets stack/reg liveness for state transitions -++ */ -++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; -++ } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { -++ /* If value_regno==-1, the caller is asking us whether -++ * it is acceptable to use this value as a SCALAR_VALUE -++ * (e.g. for XADD). -++ * We must not allow unprivileged callers to do that -++ * with spilled pointers. -++ */ -++ verbose(env, "leaking pointer from stack off %d\n", -++ off); -++ return -EACCES; -++ } -++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); -+ } else { -++ int zeros = 0; -++ -+ for (i = 0; i < size; i++) { -+- if (slot_type[i] != STACK_MISC) { -+- verbose("invalid read from stack off %d+%d size %d\n", -+- off, i, size); -+- return -EACCES; -++ if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) -++ continue; -++ if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { -++ zeros++; -++ continue; -+ } -++ verbose(env, "invalid read from stack off %d+%d size %d\n", -++ off, i, size); -++ return -EACCES; -++ } -++ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); -++ if (value_regno >= 0) { -++ if (zeros == size) { -++ /* any size read into register is zero extended, -++ * so the whole register == const_zero -++ */ -++ __mark_reg_const_zero(&state->regs[value_regno]); -++ /* backtracking doesn't support STACK_ZERO yet, -++ * so mark it precise here, so that later -++ * backtracking can stop here. -++ * Backtracking may not need this if this register -++ * doesn't participate in pointer adjustment. -++ * Forward propagation of precise flag is not -++ * necessary either. This mark is only to stop -++ * backtracking. Any register that contributed -++ * to const 0 was marked precise before spill. -++ */ -++ state->regs[value_regno].precise = true; -++ } else { -++ /* have read misc data from the stack */ -++ mark_reg_unknown(env, state->regs, value_regno); -++ } -++ state->regs[value_regno].live |= REG_LIVE_WRITTEN; -+ } -+- if (value_regno >= 0) -+- /* have read misc data from the stack */ -+- mark_reg_unknown_value(state->regs, value_regno); -+- return 0; -+ } -++ return 0; -++} -++ -++static int check_stack_access(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, -++ int off, int size) -++{ -++ /* Stack accesses must be at a fixed offset, so that we -++ * can determine what type of data were returned. See -++ * check_stack_read(). -++ */ -++ if (!tnum_is_const(reg->var_off)) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "variable stack access var_off=%s off=%d size=%d\n", -++ tn_buf, off, size); -++ return -EACCES; -++ } -++ -++ if (off >= 0 || off < -MAX_BPF_STACK) { -++ verbose(env, "invalid stack off=%d size=%d\n", off, size); -++ return -EACCES; -++ } -++ -++ return 0; -++} -++ -++static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, -++ int off, int size, enum bpf_access_type type) -++{ -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_map *map = regs[regno].map_ptr; -++ u32 cap = bpf_map_flags_to_cap(map); -++ -++ if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { -++ verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", -++ map->value_size, off, size); -++ return -EACCES; -++ } -++ -++ if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { -++ verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", -++ map->value_size, off, size); -++ return -EACCES; -++ } -++ -++ return 0; -+ } -+ -+ /* check read/write into map element returned by bpf_map_lookup_elem() */ -+-static int check_map_access(struct verifier_env *env, u32 regno, int off, -+- int size) -++static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off, -++ int size, bool zero_size_allowed) -+ { -+- struct bpf_map *map = env->cur_state.regs[regno].map_ptr; -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_map *map = regs[regno].map_ptr; -+ -+- if (off < 0 || off + size > map->value_size) { -+- verbose("invalid access to map value, value_size=%d off=%d size=%d\n", -++ if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || -++ off + size > map->value_size) { -++ verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", -+ map->value_size, off, size); -+ return -EACCES; -+ } -+ return 0; -+ } -+ -+-/* check access to 'struct bpf_context' fields */ -+-static int check_ctx_access(struct verifier_env *env, int off, int size, -+- enum bpf_access_type t) -++/* check read/write into a map element with possible variable offset */ -++static int check_map_access(struct bpf_verifier_env *env, u32 regno, -++ int off, int size, bool zero_size_allowed) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ struct bpf_reg_state *reg = &state->regs[regno]; -++ int err; -++ -++ /* We may have adjusted the register to this map value, so we -++ * need to try adding each of min_value and max_value to off -++ * to make sure our theoretical access will be safe. -++ */ -++ if (env->log.level & BPF_LOG_LEVEL) -++ print_verifier_state(env, state); -++ -++ /* The minimum value is only important with signed -++ * comparisons where we can't assume the floor of a -++ * value is 0. If we are using signed variables for our -++ * index'es we need to make sure that whatever we use -++ * will have a set floor within our range. -++ */ -++ if (reg->smin_value < 0 && -++ (reg->smin_value == S64_MIN || -++ (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || -++ reg->smin_value + off < 0)) { -++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", -++ regno); -++ return -EACCES; -++ } -++ err = __check_map_access(env, regno, reg->smin_value + off, size, -++ zero_size_allowed); -++ if (err) { -++ verbose(env, "R%d min value is outside of the array range\n", -++ regno); -++ return err; -++ } -++ -++ /* If we haven't set a max value then we need to bail since we can't be -++ * sure we won't do bad things. -++ * If reg->umax_value + off could overflow, treat that as unbounded too. -++ */ -++ if (reg->umax_value >= BPF_MAX_VAR_OFF) { -++ verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n", -++ regno); -++ return -EACCES; -++ } -++ err = __check_map_access(env, regno, reg->umax_value + off, size, -++ zero_size_allowed); -++ if (err) -++ verbose(env, "R%d max value is outside of the array range\n", -++ regno); -++ -++ if (map_value_has_spin_lock(reg->map_ptr)) { -++ u32 lock = reg->map_ptr->spin_lock_off; -++ -++ /* if any part of struct bpf_spin_lock can be touched by -++ * load/store reject this program. -++ * To check that [x1, x2) overlaps with [y1, y2) -++ * it is sufficient to check x1 < y2 && y1 < x2. -++ */ -++ if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && -++ lock < reg->umax_value + off + size) { -++ verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); -++ return -EACCES; -++ } -++ } -++ return err; -++} -++ -++#define MAX_PACKET_OFF 0xffff -++ -++static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, -++ const struct bpf_call_arg_meta *meta, -++ enum bpf_access_type t) -++{ -++ switch (env->prog->type) { -++ /* Program types only with direct read access go here! */ -++ case BPF_PROG_TYPE_LWT_IN: -++ case BPF_PROG_TYPE_LWT_OUT: -++ case BPF_PROG_TYPE_LWT_SEG6LOCAL: -++ case BPF_PROG_TYPE_SK_REUSEPORT: -++ case BPF_PROG_TYPE_FLOW_DISSECTOR: -++ case BPF_PROG_TYPE_CGROUP_SKB: -++ if (t == BPF_WRITE) -++ return false; -++ /* fallthrough */ -++ -++ /* Program types with direct read + write access go here! */ -++ case BPF_PROG_TYPE_SCHED_CLS: -++ case BPF_PROG_TYPE_SCHED_ACT: -++ case BPF_PROG_TYPE_XDP: -++ case BPF_PROG_TYPE_LWT_XMIT: -++ case BPF_PROG_TYPE_SK_SKB: -++ case BPF_PROG_TYPE_SK_MSG: -++ if (meta) -++ return meta->pkt_access; -++ -++ env->seen_direct_write = true; -++ return true; -++ -++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: -++ if (t == BPF_WRITE) -++ env->seen_direct_write = true; -++ -++ return true; -++ -++ default: -++ return false; -++ } -++} -++ -++static int __check_packet_access(struct bpf_verifier_env *env, u32 regno, -++ int off, int size, bool zero_size_allowed) -++{ -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_reg_state *reg = ®s[regno]; -++ -++ if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || -++ (u64)off + size > reg->range) { -++ verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", -++ off, size, regno, reg->id, reg->off, reg->range); -++ return -EACCES; -++ } -++ return 0; -++} -++ -++static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, -++ int size, bool zero_size_allowed) -++{ -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_reg_state *reg = ®s[regno]; -++ int err; -++ -++ /* We may have added a variable offset to the packet pointer; but any -++ * reg->range we have comes after that. We are only checking the fixed -++ * offset. -++ */ -++ -++ /* We don't allow negative numbers, because we aren't tracking enough -++ * detail to prove they're safe. -++ */ -++ if (reg->smin_value < 0) { -++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", -++ regno); -++ return -EACCES; -++ } -++ err = __check_packet_access(env, regno, off, size, zero_size_allowed); -++ if (err) { -++ verbose(env, "R%d offset is outside of the packet\n", regno); -++ return err; -++ } -++ -++ /* __check_packet_access has made sure "off + size - 1" is within u16. -++ * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, -++ * otherwise find_good_pkt_pointers would have refused to set range info -++ * that __check_packet_access would have rejected this pkt access. -++ * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. -++ */ -++ env->prog->aux->max_pkt_offset = -++ max_t(u32, env->prog->aux->max_pkt_offset, -++ off + reg->umax_value + size - 1); -++ -++ return err; -++} -++ -++/* check access to 'struct bpf_context' fields. Supports fixed offsets only */ -++static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, -++ enum bpf_access_type t, enum bpf_reg_type *reg_type) -+ { -+- if (env->prog->aux->ops->is_valid_access && -+- env->prog->aux->ops->is_valid_access(off, size, t)) -++ struct bpf_insn_access_aux info = { -++ .reg_type = *reg_type, -++ }; -++ -++ if (env->ops->is_valid_access && -++ env->ops->is_valid_access(off, size, t, env->prog, &info)) { -++ /* A non zero info.ctx_field_size indicates that this field is a -++ * candidate for later verifier transformation to load the whole -++ * field and then apply a mask when accessed with a narrower -++ * access than actual ctx access size. A zero info.ctx_field_size -++ * will only allow for whole field access and rejects any other -++ * type of narrower access. -++ */ -++ *reg_type = info.reg_type; -++ -++ env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; -++ /* remember the offset of last byte accessed in ctx */ -++ if (env->prog->aux->max_ctx_offset < off + size) -++ env->prog->aux->max_ctx_offset = off + size; -+ return 0; -++ } -+ -+- verbose("invalid bpf_context access off=%d size=%d\n", off, size); -++ verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); -+ return -EACCES; -+ } -+ -+-static bool is_pointer_value(struct verifier_env *env, int regno) -++static int check_flow_keys_access(struct bpf_verifier_env *env, int off, -++ int size) -+ { -+- if (env->allow_ptr_leaks) -+- return false; -++ if (size < 0 || off < 0 || -++ (u64)off + size > sizeof(struct bpf_flow_keys)) { -++ verbose(env, "invalid access to flow keys off=%d size=%d\n", -++ off, size); -++ return -EACCES; -++ } -++ return 0; -++} -+ -+- switch (env->cur_state.regs[regno].type) { -+- case UNKNOWN_VALUE: -+- case CONST_IMM: -+- return false; -++static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, -++ u32 regno, int off, int size, -++ enum bpf_access_type t) -++{ -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_reg_state *reg = ®s[regno]; -++ struct bpf_insn_access_aux info = {}; -++ bool valid; -++ -++ if (reg->smin_value < 0) { -++ verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", -++ regno); -++ return -EACCES; -++ } -++ -++ switch (reg->type) { -++ case PTR_TO_SOCK_COMMON: -++ valid = bpf_sock_common_is_valid_access(off, size, t, &info); -++ break; -+ default: -+- return true; -++ valid = false; -++ } -++ -++ -++ if (valid) { -++ env->insn_aux_data[insn_idx].ctx_field_size = -++ info.ctx_field_size; -++ return 0; -++ } -++ -++ verbose(env, "R%d invalid %s access off=%d size=%d\n", -++ regno, reg_type_str[reg->type], off, size); -++ -++ return -EACCES; -++} -++ -++static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) -++{ -++ return cur_regs(env) + regno; -++} -++ -++static bool is_pointer_value(struct bpf_verifier_env *env, int regno) -++{ -++ return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); -++} -++ -++static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) -++{ -++ const struct bpf_reg_state *reg = reg_state(env, regno); -++ -++ return reg->type == PTR_TO_CTX; -++} -++ -++static bool is_sk_reg(struct bpf_verifier_env *env, int regno) -++{ -++ const struct bpf_reg_state *reg = reg_state(env, regno); -++ -++ return type_is_sk_pointer(reg->type); -++} -++ -++static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) -++{ -++ const struct bpf_reg_state *reg = reg_state(env, regno); -++ -++ return type_is_pkt_pointer(reg->type); -++} -++ -++static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) -++{ -++ const struct bpf_reg_state *reg = reg_state(env, regno); -++ -++ /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ -++ return reg->type == PTR_TO_FLOW_KEYS; -++} -++ -++static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, -++ int off, int size, bool strict) -++{ -++ struct tnum reg_off; -++ int ip_align; -++ -++ /* Byte size accesses are always allowed. */ -++ if (!strict || size == 1) -++ return 0; -++ -++ /* For platforms that do not have a Kconfig enabling -++ * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of -++ * NET_IP_ALIGN is universally set to '2'. And on platforms -++ * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get -++ * to this code only in strict mode where we want to emulate -++ * the NET_IP_ALIGN==2 checking. Therefore use an -++ * unconditional IP align value of '2'. -++ */ -++ ip_align = 2; -++ -++ reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); -++ if (!tnum_is_aligned(reg_off, size)) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, -++ "misaligned packet access off %d+%s+%d+%d size %d\n", -++ ip_align, tn_buf, reg->off, off, size); -++ return -EACCES; -++ } -++ -++ return 0; -++} -++ -++static int check_generic_ptr_alignment(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, -++ const char *pointer_desc, -++ int off, int size, bool strict) -++{ -++ struct tnum reg_off; -++ -++ /* Byte size accesses are always allowed. */ -++ if (!strict || size == 1) -++ return 0; -++ -++ reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); -++ if (!tnum_is_aligned(reg_off, size)) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", -++ pointer_desc, tn_buf, reg->off, off, size); -++ return -EACCES; -++ } -++ -++ return 0; -++} -++ -++static int check_ptr_alignment(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, int off, -++ int size, bool strict_alignment_once) -++{ -++ bool strict = env->strict_alignment || strict_alignment_once; -++ const char *pointer_desc = ""; -++ -++ switch (reg->type) { -++ case PTR_TO_PACKET: -++ case PTR_TO_PACKET_META: -++ /* Special case, because of NET_IP_ALIGN. Given metadata sits -++ * right in front, treat it the very same way. -++ */ -++ return check_pkt_ptr_alignment(env, reg, off, size, strict); -++ case PTR_TO_FLOW_KEYS: -++ pointer_desc = "flow keys "; -++ break; -++ case PTR_TO_MAP_VALUE: -++ pointer_desc = "value "; -++ break; -++ case PTR_TO_CTX: -++ pointer_desc = "context "; -++ break; -++ case PTR_TO_STACK: -++ pointer_desc = "stack "; -++ /* The stack spill tracking logic in check_stack_write() -++ * and check_stack_read() relies on stack accesses being -++ * aligned. -++ */ -++ strict = true; -++ break; -++ case PTR_TO_SOCKET: -++ pointer_desc = "sock "; -++ break; -++ case PTR_TO_SOCK_COMMON: -++ pointer_desc = "sock_common "; -++ break; -++ case PTR_TO_TCP_SOCK: -++ pointer_desc = "tcp_sock "; -++ break; -++ case PTR_TO_XDP_SOCK: -++ pointer_desc = "xdp_sock "; -++ break; -++ default: -++ break; -++ } -++ return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, -++ strict); -++} -++ -++static int update_stack_depth(struct bpf_verifier_env *env, -++ const struct bpf_func_state *func, -++ int off) -++{ -++ u16 stack = env->subprog_info[func->subprogno].stack_depth; -++ -++ if (stack >= -off) -++ return 0; -++ -++ /* update known max for given subprogram */ -++ env->subprog_info[func->subprogno].stack_depth = -off; -++ return 0; -++} -++ -++/* starting from main bpf function walk all instructions of the function -++ * and recursively walk all callees that given function can call. -++ * Ignore jump and exit insns. -++ * Since recursion is prevented by check_cfg() this algorithm -++ * only needs a local stack of MAX_CALL_FRAMES to remember callsites -++ */ -++static int check_max_stack_depth(struct bpf_verifier_env *env) -++{ -++ int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; -++ struct bpf_subprog_info *subprog = env->subprog_info; -++ struct bpf_insn *insn = env->prog->insnsi; -++ int ret_insn[MAX_CALL_FRAMES]; -++ int ret_prog[MAX_CALL_FRAMES]; -++ -++process_func: -++ /* protect against potential stack overflow that might happen when -++ * bpf2bpf calls get combined with tailcalls. Limit the caller's stack -++ * depth for such case down to 256 so that the worst case scenario -++ * would result in 8k stack size (32 which is tailcall limit * 256 = -++ * 8k). -++ * -++ * To get the idea what might happen, see an example: -++ * func1 -> sub rsp, 128 -++ * subfunc1 -> sub rsp, 256 -++ * tailcall1 -> add rsp, 256 -++ * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) -++ * subfunc2 -> sub rsp, 64 -++ * subfunc22 -> sub rsp, 128 -++ * tailcall2 -> add rsp, 128 -++ * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) -++ * -++ * tailcall will unwind the current stack frame but it will not get rid -++ * of caller's stack as shown on the example above. -++ */ -++ if (idx && subprog[idx].has_tail_call && depth >= 256) { -++ verbose(env, -++ "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", -++ depth); -++ return -EACCES; -++ } -++ /* round up to 32-bytes, since this is granularity -++ * of interpreter stack size -++ */ -++ depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); -++ if (depth > MAX_BPF_STACK) { -++ verbose(env, "combined stack size of %d calls is %d. Too large\n", -++ frame + 1, depth); -++ return -EACCES; -++ } -++continue_func: -++ subprog_end = subprog[idx + 1].start; -++ for (; i < subprog_end; i++) { -++ if (insn[i].code != (BPF_JMP | BPF_CALL)) -++ continue; -++ if (insn[i].src_reg != BPF_PSEUDO_CALL) -++ continue; -++ /* remember insn and function to return to */ -++ ret_insn[frame] = i + 1; -++ ret_prog[frame] = idx; -++ -++ /* find the callee */ -++ i = i + insn[i].imm + 1; -++ idx = find_subprog(env, i); -++ if (idx < 0) { -++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", -++ i); -++ return -EFAULT; -++ } -++ frame++; -++ if (frame >= MAX_CALL_FRAMES) { -++ verbose(env, "the call stack of %d frames is too deep !\n", -++ frame); -++ return -E2BIG; -++ } -++ goto process_func; -++ } -++ /* end of for() loop means the last insn of the 'subprog' -++ * was reached. Doesn't matter whether it was JA or EXIT -++ */ -++ if (frame == 0) -++ return 0; -++ depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); -++ frame--; -++ i = ret_insn[frame]; -++ idx = ret_prog[frame]; -++ goto continue_func; -++} -++ -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++static int get_callee_stack_depth(struct bpf_verifier_env *env, -++ const struct bpf_insn *insn, int idx) -++{ -++ int start = idx + insn->imm + 1, subprog; -++ -++ subprog = find_subprog(env, start); -++ if (subprog < 0) { -++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", -++ start); -++ return -EFAULT; -+ } -++ return env->subprog_info[subprog].stack_depth; -++} -++#endif -++ -++static int check_ctx_reg(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, int regno) -++{ -++ /* Access to ctx or passing it to a helper is only allowed in -++ * its original, unmodified form. -++ */ -++ -++ if (reg->off) { -++ verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", -++ regno, reg->off); -++ return -EACCES; -++ } -++ -++ if (!tnum_is_const(reg->var_off) || reg->var_off.value) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); -++ return -EACCES; -++ } -++ -++ return 0; -++} -++ -++static int check_tp_buffer_access(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, -++ int regno, int off, int size) -++{ -++ if (off < 0) { -++ verbose(env, -++ "R%d invalid tracepoint buffer access: off=%d, size=%d", -++ regno, off, size); -++ return -EACCES; -++ } -++ if (!tnum_is_const(reg->var_off) || reg->var_off.value) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, -++ "R%d invalid variable buffer offset: off=%d, var_off=%s", -++ regno, off, tn_buf); -++ return -EACCES; -++ } -++ if (off + size > env->prog->aux->max_tp_access) -++ env->prog->aux->max_tp_access = off + size; -++ -++ return 0; -++} -++ -++ -++/* truncate register to smaller size (in bytes) -++ * must be called with size < BPF_REG_SIZE -++ */ -++static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) -++{ -++ u64 mask; -++ -++ /* clear high bits in bit representation */ -++ reg->var_off = tnum_cast(reg->var_off, size); -++ -++ /* fix arithmetic bounds */ -++ mask = ((u64)1 << (size * 8)) - 1; -++ if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { -++ reg->umin_value &= mask; -++ reg->umax_value &= mask; -++ } else { -++ reg->umin_value = 0; -++ reg->umax_value = mask; -++ } -++ reg->smin_value = reg->umin_value; -++ reg->smax_value = reg->umax_value; -+ } -+ -+ /* check whether memory at (regno + off) is accessible for t = (read | write) -+@@ -680,225 +2776,692 @@ static bool is_pointer_value(struct veri -+ * if t==write && value_regno==-1, some unknown value is stored into memory -+ * if t==read && value_regno==-1, don't care what we read from memory -+ */ -+-static int check_mem_access(struct verifier_env *env, u32 regno, int off, -+- int bpf_size, enum bpf_access_type t, -+- int value_regno) -++static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, -++ int off, int bpf_size, enum bpf_access_type t, -++ int value_regno, bool strict_alignment_once) -+ { -+- struct verifier_state *state = &env->cur_state; -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_reg_state *reg = regs + regno; -++ struct bpf_func_state *state; -+ int size, err = 0; -+ -+- if (state->regs[regno].type == PTR_TO_STACK) -+- off += state->regs[regno].imm; -+- -+ size = bpf_size_to_bytes(bpf_size); -+ if (size < 0) -+ return size; -+ -+- if (off % size != 0) { -+- verbose("misaligned access off %d size %d\n", off, size); -+- return -EACCES; -+- } -++ /* alignment checks will add in reg->off themselves */ -++ err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); -++ if (err) -++ return err; -++ -++ /* for access checks, reg->off is just part of off */ -++ off += reg->off; -+ -+- if (state->regs[regno].type == PTR_TO_MAP_VALUE) { -++ if (reg->type == PTR_TO_MAP_VALUE) { -+ if (t == BPF_WRITE && value_regno >= 0 && -+ is_pointer_value(env, value_regno)) { -+- verbose("R%d leaks addr into map\n", value_regno); -++ verbose(env, "R%d leaks addr into map\n", value_regno); -+ return -EACCES; -+ } -+- err = check_map_access(env, regno, off, size); -++ err = check_map_access_type(env, regno, off, size, t); -++ if (err) -++ return err; -++ err = check_map_access(env, regno, off, size, false); -+ if (!err && t == BPF_READ && value_regno >= 0) -+- mark_reg_unknown_value(state->regs, value_regno); -++ mark_reg_unknown(env, regs, value_regno); -++ -++ } else if (reg->type == PTR_TO_CTX) { -++ enum bpf_reg_type reg_type = SCALAR_VALUE; -+ -+- } else if (state->regs[regno].type == PTR_TO_CTX) { -+ if (t == BPF_WRITE && value_regno >= 0 && -+ is_pointer_value(env, value_regno)) { -+- verbose("R%d leaks addr into ctx\n", value_regno); -++ verbose(env, "R%d leaks addr into ctx\n", value_regno); -+ return -EACCES; -+ } -+- err = check_ctx_access(env, off, size, t); -+- if (!err && t == BPF_READ && value_regno >= 0) -+- mark_reg_unknown_value(state->regs, value_regno); -+ -+- } else if (state->regs[regno].type == FRAME_PTR || -+- state->regs[regno].type == PTR_TO_STACK) { -+- if (off >= 0 || off < -MAX_BPF_STACK) { -+- verbose("invalid stack off=%d size=%d\n", off, size); -++ err = check_ctx_reg(env, reg, regno); -++ if (err < 0) -++ return err; -++ -++ err = check_ctx_access(env, insn_idx, off, size, t, ®_type); -++ if (!err && t == BPF_READ && value_regno >= 0) { -++ /* ctx access returns either a scalar, or a -++ * PTR_TO_PACKET[_META,_END]. In the latter -++ * case, we know the offset is zero. -++ */ -++ if (reg_type == SCALAR_VALUE) { -++ mark_reg_unknown(env, regs, value_regno); -++ } else { -++ mark_reg_known_zero(env, regs, -++ value_regno); -++ if (reg_type_may_be_null(reg_type)) -++ regs[value_regno].id = ++env->id_gen; -++ /* A load of ctx field could have different -++ * actual load size with the one encoded in the -++ * insn. When the dst is PTR, it is for sure not -++ * a sub-register. -++ */ -++ regs[value_regno].subreg_def = DEF_NOT_SUBREG; -++ } -++ regs[value_regno].type = reg_type; -++ } -++ -++ } else if (reg->type == PTR_TO_STACK) { -++ off += reg->var_off.value; -++ err = check_stack_access(env, reg, off, size); -++ if (err) -++ return err; -++ -++ state = func(env, reg); -++ err = update_stack_depth(env, state, off); -++ if (err) -++ return err; -++ -++ if (t == BPF_WRITE) -++ err = check_stack_write(env, state, off, size, -++ value_regno, insn_idx); -++ else -++ err = check_stack_read(env, state, off, size, -++ value_regno); -++ } else if (reg_is_pkt_pointer(reg)) { -++ if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { -++ verbose(env, "cannot write into packet\n"); -++ return -EACCES; -++ } -++ if (t == BPF_WRITE && value_regno >= 0 && -++ is_pointer_value(env, value_regno)) { -++ verbose(env, "R%d leaks addr into packet\n", -++ value_regno); -++ return -EACCES; -++ } -++ err = check_packet_access(env, regno, off, size, false); -++ if (!err && t == BPF_READ && value_regno >= 0) -++ mark_reg_unknown(env, regs, value_regno); -++ } else if (reg->type == PTR_TO_FLOW_KEYS) { -++ if (t == BPF_WRITE && value_regno >= 0 && -++ is_pointer_value(env, value_regno)) { -++ verbose(env, "R%d leaks addr into flow keys\n", -++ value_regno); -+ return -EACCES; -+ } -++ -++ err = check_flow_keys_access(env, off, size); -++ if (!err && t == BPF_READ && value_regno >= 0) -++ mark_reg_unknown(env, regs, value_regno); -++ } else if (type_is_sk_pointer(reg->type)) { -+ if (t == BPF_WRITE) { -+- if (!env->allow_ptr_leaks && -+- state->stack_slot_type[MAX_BPF_STACK + off] == STACK_SPILL && -+- size != BPF_REG_SIZE) { -+- verbose("attempt to corrupt spilled pointer on stack\n"); -+- return -EACCES; -+- } -+- err = check_stack_write(state, off, size, value_regno); -+- } else { -+- err = check_stack_read(state, off, size, value_regno); -++ verbose(env, "R%d cannot write into %s\n", -++ regno, reg_type_str[reg->type]); -++ return -EACCES; -+ } -++ err = check_sock_access(env, insn_idx, regno, off, size, t); -++ if (!err && value_regno >= 0) -++ mark_reg_unknown(env, regs, value_regno); -++ } else if (reg->type == PTR_TO_TP_BUFFER) { -++ err = check_tp_buffer_access(env, reg, regno, off, size); -++ if (!err && t == BPF_READ && value_regno >= 0) -++ mark_reg_unknown(env, regs, value_regno); -+ } else { -+- verbose("R%d invalid mem access '%s'\n", -+- regno, reg_type_str[state->regs[regno].type]); -++ verbose(env, "R%d invalid mem access '%s'\n", regno, -++ reg_type_str[reg->type]); -+ return -EACCES; -+ } -++ -++ if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && -++ regs[value_regno].type == SCALAR_VALUE) { -++ /* b/h/w load zero-extends, mark upper bits as known 0 */ -++ coerce_reg_to_size(®s[value_regno], size); -++ } -+ return err; -+ } -+ -+-static int check_xadd(struct verifier_env *env, struct bpf_insn *insn) -++static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) -+ { -+- struct reg_state *regs = env->cur_state.regs; -+ int err; -+ -+ if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || -+ insn->imm != 0) { -+- verbose("BPF_XADD uses reserved fields\n"); -++ verbose(env, "BPF_XADD uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ /* check src1 operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ -+ /* check src2 operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -++ if (is_pointer_value(env, insn->src_reg)) { -++ verbose(env, "R%d leaks addr into mem\n", insn->src_reg); -++ return -EACCES; -++ } -++ -++ if (is_ctx_reg(env, insn->dst_reg) || -++ is_pkt_reg(env, insn->dst_reg) || -++ is_flow_key_reg(env, insn->dst_reg) || -++ is_sk_reg(env, insn->dst_reg)) { -++ verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", -++ insn->dst_reg, -++ reg_type_str[reg_state(env, insn->dst_reg)->type]); -++ return -EACCES; -++ } -++ -+ /* check whether atomic_add can read the memory */ -+- err = check_mem_access(env, insn->dst_reg, insn->off, -+- BPF_SIZE(insn->code), BPF_READ, -1); -++ err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, -++ BPF_SIZE(insn->code), BPF_READ, -1, true); -+ if (err) -+ return err; -+ -+ /* check whether atomic_add can write into the same memory */ -+- return check_mem_access(env, insn->dst_reg, insn->off, -+- BPF_SIZE(insn->code), BPF_WRITE, -1); -++ return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, -++ BPF_SIZE(insn->code), BPF_WRITE, -1, true); -++} -++ -++static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, -++ int off, int access_size, -++ bool zero_size_allowed) -++{ -++ struct bpf_reg_state *reg = reg_state(env, regno); -++ -++ if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || -++ access_size < 0 || (access_size == 0 && !zero_size_allowed)) { -++ if (tnum_is_const(reg->var_off)) { -++ verbose(env, "invalid stack type R%d off=%d access_size=%d\n", -++ regno, off, access_size); -++ } else { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", -++ regno, tn_buf, access_size); -++ } -++ return -EACCES; -++ } -++ return 0; -+ } -+ -+ /* when register 'regno' is passed into function that will read 'access_size' -+ * bytes from that pointer, make sure that it's within stack boundary -+- * and all elements of stack are initialized -++ * and all elements of stack are initialized. -++ * Unlike most pointer bounds-checking functions, this one doesn't take an -++ * 'off' argument, so it has to add in reg->off itself. -+ */ -+-static int check_stack_boundary(struct verifier_env *env, -+- int regno, int access_size) -++static int check_stack_boundary(struct bpf_verifier_env *env, int regno, -++ int access_size, bool zero_size_allowed, -++ struct bpf_call_arg_meta *meta) -+ { -+- struct verifier_state *state = &env->cur_state; -+- struct reg_state *regs = state->regs; -+- int off, i; -++ struct bpf_reg_state *reg = reg_state(env, regno); -++ struct bpf_func_state *state = func(env, reg); -++ int err, min_off, max_off, i, j, slot, spi; -++ -++ if (reg->type != PTR_TO_STACK) { -++ /* Allow zero-byte read from NULL, regardless of pointer type */ -++ if (zero_size_allowed && access_size == 0 && -++ register_is_null(reg)) -++ return 0; -+ -+- if (regs[regno].type != PTR_TO_STACK) -++ verbose(env, "R%d type=%s expected=%s\n", regno, -++ reg_type_str[reg->type], -++ reg_type_str[PTR_TO_STACK]); -+ return -EACCES; -++ } -+ -+- off = regs[regno].imm; -+- if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || -+- access_size <= 0) { -+- verbose("invalid stack type R%d off=%d access_size=%d\n", -+- regno, off, access_size); -++ if (tnum_is_const(reg->var_off)) { -++ min_off = max_off = reg->var_off.value + reg->off; -++ err = __check_stack_boundary(env, regno, min_off, access_size, -++ zero_size_allowed); -++ if (err) -++ return err; -++ } else { -++ /* Variable offset is prohibited for unprivileged mode for -++ * simplicity since it requires corresponding support in -++ * Spectre masking for stack ALU. -++ * See also retrieve_ptr_limit(). -++ */ -++ if (!env->allow_ptr_leaks) { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", -++ regno, tn_buf); -++ return -EACCES; -++ } -++ /* Only initialized buffer on stack is allowed to be accessed -++ * with variable offset. With uninitialized buffer it's hard to -++ * guarantee that whole memory is marked as initialized on -++ * helper return since specific bounds are unknown what may -++ * cause uninitialized stack leaking. -++ */ -++ if (meta && meta->raw_mode) -++ meta = NULL; -++ -++ if (reg->smax_value >= BPF_MAX_VAR_OFF || -++ reg->smax_value <= -BPF_MAX_VAR_OFF) { -++ verbose(env, "R%d unbounded indirect variable offset stack access\n", -++ regno); -++ return -EACCES; -++ } -++ min_off = reg->smin_value + reg->off; -++ max_off = reg->smax_value + reg->off; -++ err = __check_stack_boundary(env, regno, min_off, access_size, -++ zero_size_allowed); -++ if (err) { -++ verbose(env, "R%d min value is outside of stack bound\n", -++ regno); -++ return err; -++ } -++ err = __check_stack_boundary(env, regno, max_off, access_size, -++ zero_size_allowed); -++ if (err) { -++ verbose(env, "R%d max value is outside of stack bound\n", -++ regno); -++ return err; -++ } -++ } -++ -++ if (meta && meta->raw_mode) { -++ meta->access_size = access_size; -++ meta->regno = regno; -++ return 0; -++ } -++ -++ for (i = min_off; i < max_off + access_size; i++) { -++ u8 *stype; -++ -++ slot = -i - 1; -++ spi = slot / BPF_REG_SIZE; -++ if (state->allocated_stack <= slot) -++ goto err; -++ stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; -++ if (*stype == STACK_MISC) -++ goto mark; -++ if (*stype == STACK_ZERO) { -++ /* helper can write anything into the stack */ -++ *stype = STACK_MISC; -++ goto mark; -++ } -++ if (state->stack[spi].slot_type[0] == STACK_SPILL && -++ state->stack[spi].spilled_ptr.type == SCALAR_VALUE) { -++ __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); -++ for (j = 0; j < BPF_REG_SIZE; j++) -++ state->stack[spi].slot_type[j] = STACK_MISC; -++ goto mark; -++ } -++ -++err: -++ if (tnum_is_const(reg->var_off)) { -++ verbose(env, "invalid indirect read from stack off %d+%d size %d\n", -++ min_off, i - min_off, access_size); -++ } else { -++ char tn_buf[48]; -++ -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", -++ tn_buf, i - min_off, access_size); -++ } -+ return -EACCES; -++mark: -++ /* reading any byte out of 8-byte 'spill_slot' will cause -++ * the whole slot to be marked as 'read' -++ */ -++ mark_reg_read(env, &state->stack[spi].spilled_ptr, -++ state->stack[spi].spilled_ptr.parent, -++ REG_LIVE_READ64); -+ } -++ return update_stack_depth(env, state, min_off); -++} -++ -++static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, -++ int access_size, bool zero_size_allowed, -++ struct bpf_call_arg_meta *meta) -++{ -++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; -+ -+- for (i = 0; i < access_size; i++) { -+- if (state->stack_slot_type[MAX_BPF_STACK + off + i] != STACK_MISC) { -+- verbose("invalid indirect read from stack off %d+%d size %d\n", -+- off, i, access_size); -++ switch (reg->type) { -++ case PTR_TO_PACKET: -++ case PTR_TO_PACKET_META: -++ return check_packet_access(env, regno, reg->off, access_size, -++ zero_size_allowed); -++ case PTR_TO_MAP_VALUE: -++ if (check_map_access_type(env, regno, reg->off, access_size, -++ meta && meta->raw_mode ? BPF_WRITE : -++ BPF_READ)) -+ return -EACCES; -++ return check_map_access(env, regno, reg->off, access_size, -++ zero_size_allowed); -++ default: /* scalar_value|ptr_to_stack or invalid ptr */ -++ return check_stack_boundary(env, regno, access_size, -++ zero_size_allowed, meta); -++ } -++} -++ -++/* Implementation details: -++ * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL -++ * Two bpf_map_lookups (even with the same key) will have different reg->id. -++ * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after -++ * value_or_null->value transition, since the verifier only cares about -++ * the range of access to valid map value pointer and doesn't care about actual -++ * address of the map element. -++ * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps -++ * reg->id > 0 after value_or_null->value transition. By doing so -++ * two bpf_map_lookups will be considered two different pointers that -++ * point to different bpf_spin_locks. -++ * The verifier allows taking only one bpf_spin_lock at a time to avoid -++ * dead-locks. -++ * Since only one bpf_spin_lock is allowed the checks are simpler than -++ * reg_is_refcounted() logic. The verifier needs to remember only -++ * one spin_lock instead of array of acquired_refs. -++ * cur_state->active_spin_lock remembers which map value element got locked -++ * and clears it after bpf_spin_unlock. -++ */ -++static int process_spin_lock(struct bpf_verifier_env *env, int regno, -++ bool is_lock) -++{ -++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; -++ struct bpf_verifier_state *cur = env->cur_state; -++ bool is_const = tnum_is_const(reg->var_off); -++ struct bpf_map *map = reg->map_ptr; -++ u64 val = reg->var_off.value; -++ -++ if (reg->type != PTR_TO_MAP_VALUE) { -++ verbose(env, "R%d is not a pointer to map_value\n", regno); -++ return -EINVAL; -++ } -++ if (!is_const) { -++ verbose(env, -++ "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", -++ regno); -++ return -EINVAL; -++ } -++ if (!map->btf) { -++ verbose(env, -++ "map '%s' has to have BTF in order to use bpf_spin_lock\n", -++ map->name); -++ return -EINVAL; -++ } -++ if (!map_value_has_spin_lock(map)) { -++ if (map->spin_lock_off == -E2BIG) -++ verbose(env, -++ "map '%s' has more than one 'struct bpf_spin_lock'\n", -++ map->name); -++ else if (map->spin_lock_off == -ENOENT) -++ verbose(env, -++ "map '%s' doesn't have 'struct bpf_spin_lock'\n", -++ map->name); -++ else -++ verbose(env, -++ "map '%s' is not a struct type or bpf_spin_lock is mangled\n", -++ map->name); -++ return -EINVAL; -++ } -++ if (map->spin_lock_off != val + reg->off) { -++ verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", -++ val + reg->off); -++ return -EINVAL; -++ } -++ if (is_lock) { -++ if (cur->active_spin_lock) { -++ verbose(env, -++ "Locking two bpf_spin_locks are not allowed\n"); -++ return -EINVAL; -++ } -++ cur->active_spin_lock = reg->id; -++ } else { -++ if (!cur->active_spin_lock) { -++ verbose(env, "bpf_spin_unlock without taking a lock\n"); -++ return -EINVAL; -+ } -++ if (cur->active_spin_lock != reg->id) { -++ verbose(env, "bpf_spin_unlock of different lock\n"); -++ return -EINVAL; -++ } -++ cur->active_spin_lock = 0; -+ } -+ return 0; -+ } -+ -+-static int check_func_arg(struct verifier_env *env, u32 regno, -+- enum bpf_arg_type arg_type, struct bpf_map **mapp) -++static bool arg_type_is_mem_ptr(enum bpf_arg_type type) -++{ -++ return type == ARG_PTR_TO_MEM || -++ type == ARG_PTR_TO_MEM_OR_NULL || -++ type == ARG_PTR_TO_UNINIT_MEM; -++} -++ -++static bool arg_type_is_mem_size(enum bpf_arg_type type) -++{ -++ return type == ARG_CONST_SIZE || -++ type == ARG_CONST_SIZE_OR_ZERO; -++} -++ -++static bool arg_type_is_int_ptr(enum bpf_arg_type type) -++{ -++ return type == ARG_PTR_TO_INT || -++ type == ARG_PTR_TO_LONG; -++} -++ -++static int int_ptr_type_to_size(enum bpf_arg_type type) -++{ -++ if (type == ARG_PTR_TO_INT) -++ return sizeof(u32); -++ else if (type == ARG_PTR_TO_LONG) -++ return sizeof(u64); -++ -++ return -EINVAL; -++} -++ -++static int check_func_arg(struct bpf_verifier_env *env, u32 regno, -++ enum bpf_arg_type arg_type, -++ struct bpf_call_arg_meta *meta) -+ { -+- struct reg_state *reg = env->cur_state.regs + regno; -+- enum bpf_reg_type expected_type; -++ struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; -++ enum bpf_reg_type expected_type, type = reg->type; -+ int err = 0; -+ -+ if (arg_type == ARG_DONTCARE) -+ return 0; -+ -+- if (reg->type == NOT_INIT) { -+- verbose("R%d !read_ok\n", regno); -+- return -EACCES; -+- } -++ err = check_reg_arg(env, regno, SRC_OP); -++ if (err) -++ return err; -+ -+ if (arg_type == ARG_ANYTHING) { -+ if (is_pointer_value(env, regno)) { -+- verbose("R%d leaks addr into helper function\n", regno); -++ verbose(env, "R%d leaks addr into helper function\n", -++ regno); -+ return -EACCES; -+ } -+ return 0; -+ } -+ -+- if (arg_type == ARG_PTR_TO_STACK || arg_type == ARG_PTR_TO_MAP_KEY || -+- arg_type == ARG_PTR_TO_MAP_VALUE) { -++ if (type_is_pkt_pointer(type) && -++ !may_access_direct_pkt_data(env, meta, BPF_READ)) { -++ verbose(env, "helper access to the packet is not allowed\n"); -++ return -EACCES; -++ } -++ -++ if (arg_type == ARG_PTR_TO_MAP_KEY || -++ arg_type == ARG_PTR_TO_MAP_VALUE || -++ arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || -++ arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { -+ expected_type = PTR_TO_STACK; -+- } else if (arg_type == ARG_CONST_STACK_SIZE) { -+- expected_type = CONST_IMM; -++ if (register_is_null(reg) && -++ arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) -++ /* final test in check_stack_boundary() */; -++ else if (!type_is_pkt_pointer(type) && -++ type != PTR_TO_MAP_VALUE && -++ type != expected_type) -++ goto err_type; -++ } else if (arg_type == ARG_CONST_SIZE || -++ arg_type == ARG_CONST_SIZE_OR_ZERO) { -++ expected_type = SCALAR_VALUE; -++ if (type != expected_type) -++ goto err_type; -+ } else if (arg_type == ARG_CONST_MAP_PTR) { -+ expected_type = CONST_PTR_TO_MAP; -++ if (type != expected_type) -++ goto err_type; -+ } else if (arg_type == ARG_PTR_TO_CTX) { -+ expected_type = PTR_TO_CTX; -++ if (type != expected_type) -++ goto err_type; -++ err = check_ctx_reg(env, reg, regno); -++ if (err < 0) -++ return err; -++ } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) { -++ expected_type = PTR_TO_SOCK_COMMON; -++ /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */ -++ if (!type_is_sk_pointer(type)) -++ goto err_type; -++ if (reg->ref_obj_id) { -++ if (meta->ref_obj_id) { -++ verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", -++ regno, reg->ref_obj_id, -++ meta->ref_obj_id); -++ return -EFAULT; -++ } -++ meta->ref_obj_id = reg->ref_obj_id; -++ } -++ } else if (arg_type == ARG_PTR_TO_SOCKET) { -++ expected_type = PTR_TO_SOCKET; -++ if (type != expected_type) -++ goto err_type; -++ } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { -++ if (meta->func_id == BPF_FUNC_spin_lock) { -++ if (process_spin_lock(env, regno, true)) -++ return -EACCES; -++ } else if (meta->func_id == BPF_FUNC_spin_unlock) { -++ if (process_spin_lock(env, regno, false)) -++ return -EACCES; -++ } else { -++ verbose(env, "verifier internal error\n"); -++ return -EFAULT; -++ } -++ } else if (arg_type_is_mem_ptr(arg_type)) { -++ expected_type = PTR_TO_STACK; -++ /* One exception here. In case function allows for NULL to be -++ * passed in as argument, it's a SCALAR_VALUE type. Final test -++ * happens during stack boundary checking. -++ */ -++ if (register_is_null(reg) && -++ arg_type == ARG_PTR_TO_MEM_OR_NULL) -++ /* final test in check_stack_boundary() */; -++ else if (!type_is_pkt_pointer(type) && -++ type != PTR_TO_MAP_VALUE && -++ type != expected_type) -++ goto err_type; -++ meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM; -++ } else if (arg_type_is_int_ptr(arg_type)) { -++ expected_type = PTR_TO_STACK; -++ if (!type_is_pkt_pointer(type) && -++ type != PTR_TO_MAP_VALUE && -++ type != expected_type) -++ goto err_type; -+ } else { -+- verbose("unsupported arg_type %d\n", arg_type); -++ verbose(env, "unsupported arg_type %d\n", arg_type); -+ return -EFAULT; -+ } -+ -+- if (reg->type != expected_type) { -+- verbose("R%d type=%s expected=%s\n", regno, -+- reg_type_str[reg->type], reg_type_str[expected_type]); -+- return -EACCES; -+- } -+- -+ if (arg_type == ARG_CONST_MAP_PTR) { -+ /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ -+- *mapp = reg->map_ptr; -+- -++ meta->map_ptr = reg->map_ptr; -+ } else if (arg_type == ARG_PTR_TO_MAP_KEY) { -+ /* bpf_map_xxx(..., map_ptr, ..., key) call: -+ * check that [key, key + map->key_size) are within -+ * stack limits and initialized -+ */ -+- if (!*mapp) { -++ if (!meta->map_ptr) { -+ /* in function declaration map_ptr must come before -+ * map_key, so that it's verified and known before -+ * we have to check map_key here. Otherwise it means -+ * that kernel subsystem misconfigured verifier -+ */ -+- verbose("invalid map_ptr to access map->key\n"); -++ verbose(env, "invalid map_ptr to access map->key\n"); -+ return -EACCES; -+ } -+- err = check_stack_boundary(env, regno, (*mapp)->key_size); -+- -+- } else if (arg_type == ARG_PTR_TO_MAP_VALUE) { -++ err = check_helper_mem_access(env, regno, -++ meta->map_ptr->key_size, false, -++ NULL); -++ } else if (arg_type == ARG_PTR_TO_MAP_VALUE || -++ (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && -++ !register_is_null(reg)) || -++ arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { -+ /* bpf_map_xxx(..., map_ptr, ..., value) call: -+ * check [value, value + map->value_size) validity -+ */ -+- if (!*mapp) { -++ if (!meta->map_ptr) { -+ /* kernel subsystem misconfigured verifier */ -+- verbose("invalid map_ptr to access map->value\n"); -++ verbose(env, "invalid map_ptr to access map->value\n"); -+ return -EACCES; -+ } -+- err = check_stack_boundary(env, regno, (*mapp)->value_size); -++ meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); -++ err = check_helper_mem_access(env, regno, -++ meta->map_ptr->value_size, false, -++ meta); -++ } else if (arg_type_is_mem_size(arg_type)) { -++ bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); -+ -+- } else if (arg_type == ARG_CONST_STACK_SIZE) { -+- /* bpf_xxx(..., buf, len) call will access 'len' bytes -+- * from stack pointer 'buf'. Check it -+- * note: regno == len, regno - 1 == buf -++ /* remember the mem_size which may be used later -++ * to refine return values. -+ */ -+- if (regno == 0) { -+- /* kernel subsystem misconfigured verifier */ -+- verbose("ARG_CONST_STACK_SIZE cannot be first argument\n"); -++ meta->msize_max_value = reg->umax_value; -++ -++ /* The register is SCALAR_VALUE; the access check -++ * happens using its boundaries. -++ */ -++ if (!tnum_is_const(reg->var_off)) -++ /* For unprivileged variable accesses, disable raw -++ * mode so that the program is required to -++ * initialize all the memory that the helper could -++ * just partially fill up. -++ */ -++ meta = NULL; -++ -++ if (reg->smin_value < 0) { -++ verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", -++ regno); -++ return -EACCES; -++ } -++ -++ if (reg->umin_value == 0) { -++ err = check_helper_mem_access(env, regno - 1, 0, -++ zero_size_allowed, -++ meta); -++ if (err) -++ return err; -++ } -++ -++ if (reg->umax_value >= BPF_MAX_VAR_SIZ) { -++ verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", -++ regno); -+ return -EACCES; -+ } -+- err = check_stack_boundary(env, regno - 1, reg->imm); -++ err = check_helper_mem_access(env, regno - 1, -++ reg->umax_value, -++ zero_size_allowed, meta); -++ if (!err) -++ err = mark_chain_precision(env, regno); -++ } else if (arg_type_is_int_ptr(arg_type)) { -++ int size = int_ptr_type_to_size(arg_type); -++ -++ err = check_helper_mem_access(env, regno, size, false, meta); -++ if (err) -++ return err; -++ err = check_ptr_alignment(env, reg, 0, size, true); -+ } -+ -+ return err; -++err_type: -++ verbose(env, "R%d type=%s expected=%s\n", regno, -++ reg_type_str[type], reg_type_str[expected_type]); -++ return -EACCES; -+ } -+ -+-static int check_map_func_compatibility(struct bpf_map *map, int func_id) -++static int check_map_func_compatibility(struct bpf_verifier_env *env, -++ struct bpf_map *map, int func_id) -+ { -+ if (!map) -+ return 0; -+@@ -911,7 +3474,74 @@ static int check_map_func_compatibility( -+ break; -+ case BPF_MAP_TYPE_PERF_EVENT_ARRAY: -+ if (func_id != BPF_FUNC_perf_event_read && -+- func_id != BPF_FUNC_perf_event_output) -++ func_id != BPF_FUNC_perf_event_output && -++ func_id != BPF_FUNC_perf_event_read_value) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_STACK_TRACE: -++ if (func_id != BPF_FUNC_get_stackid) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_CGROUP_ARRAY: -++ if (func_id != BPF_FUNC_skb_under_cgroup && -++ func_id != BPF_FUNC_current_task_under_cgroup) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_CGROUP_STORAGE: -++ if (func_id != BPF_FUNC_get_local_storage) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_DEVMAP: -++ case BPF_MAP_TYPE_DEVMAP_HASH: -++ if (func_id != BPF_FUNC_redirect_map && -++ func_id != BPF_FUNC_map_lookup_elem) -++ goto error; -++ break; -++ /* Restrict bpf side of cpumap and xskmap, open when use-cases -++ * appear. -++ */ -++ case BPF_MAP_TYPE_CPUMAP: -++ if (func_id != BPF_FUNC_redirect_map) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_XSKMAP: -++ if (func_id != BPF_FUNC_redirect_map && -++ func_id != BPF_FUNC_map_lookup_elem) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_ARRAY_OF_MAPS: -++ case BPF_MAP_TYPE_HASH_OF_MAPS: -++ if (func_id != BPF_FUNC_map_lookup_elem) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_SOCKMAP: -++ if (func_id != BPF_FUNC_sk_redirect_map && -++ func_id != BPF_FUNC_sock_map_update && -++ func_id != BPF_FUNC_map_delete_elem && -++ func_id != BPF_FUNC_msg_redirect_map) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_SOCKHASH: -++ if (func_id != BPF_FUNC_sk_redirect_hash && -++ func_id != BPF_FUNC_sock_hash_update && -++ func_id != BPF_FUNC_map_delete_elem && -++ func_id != BPF_FUNC_msg_redirect_hash) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: -++ if (func_id != BPF_FUNC_sk_select_reuseport) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_QUEUE: -++ case BPF_MAP_TYPE_STACK: -++ if (func_id != BPF_FUNC_map_peek_elem && -++ func_id != BPF_FUNC_map_pop_elem && -++ func_id != BPF_FUNC_map_push_elem) -++ goto error; -++ break; -++ case BPF_MAP_TYPE_SK_STORAGE: -++ if (func_id != BPF_FUNC_sk_storage_get && -++ func_id != BPF_FUNC_sk_storage_delete) -+ goto error; -+ break; -+ default: -+@@ -923,109 +3553,1579 @@ static int check_map_func_compatibility( -+ case BPF_FUNC_tail_call: -+ if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) -+ goto error; -++ if (env->subprog_cnt > 1) { -++ verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n"); -++ return -EINVAL; -++ } -+ break; -+ case BPF_FUNC_perf_event_read: -+ case BPF_FUNC_perf_event_output: -++ case BPF_FUNC_perf_event_read_value: -+ if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) -+ goto error; -+ break; -++ case BPF_FUNC_get_stackid: -++ if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) -++ goto error; -++ break; -++ case BPF_FUNC_current_task_under_cgroup: -++ case BPF_FUNC_skb_under_cgroup: -++ if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) -++ goto error; -++ break; -++ case BPF_FUNC_redirect_map: -++ if (map->map_type != BPF_MAP_TYPE_DEVMAP && -++ map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && -++ map->map_type != BPF_MAP_TYPE_CPUMAP && -++ map->map_type != BPF_MAP_TYPE_XSKMAP) -++ goto error; -++ break; -++ case BPF_FUNC_sk_redirect_map: -++ case BPF_FUNC_msg_redirect_map: -++ case BPF_FUNC_sock_map_update: -++ if (map->map_type != BPF_MAP_TYPE_SOCKMAP) -++ goto error; -++ break; -++ case BPF_FUNC_sk_redirect_hash: -++ case BPF_FUNC_msg_redirect_hash: -++ case BPF_FUNC_sock_hash_update: -++ if (map->map_type != BPF_MAP_TYPE_SOCKHASH) -++ goto error; -++ break; -++ case BPF_FUNC_get_local_storage: -++ if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && -++ map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) -++ goto error; -++ break; -++ case BPF_FUNC_sk_select_reuseport: -++ if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) -++ goto error; -++ break; -++ case BPF_FUNC_map_peek_elem: -++ case BPF_FUNC_map_pop_elem: -++ case BPF_FUNC_map_push_elem: -++ if (map->map_type != BPF_MAP_TYPE_QUEUE && -++ map->map_type != BPF_MAP_TYPE_STACK) -++ goto error; -++ break; -++ case BPF_FUNC_sk_storage_get: -++ case BPF_FUNC_sk_storage_delete: -++ if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) -++ goto error; -++ break; -+ default: -+ break; -+ } -+ -+ return 0; -+ error: -+- verbose("cannot pass map_type %d into func %d\n", -+- map->map_type, func_id); -++ verbose(env, "cannot pass map_type %d into func %s#%d\n", -++ map->map_type, func_id_name(func_id), func_id); -+ return -EINVAL; -+ } -+ -+-static int check_call(struct verifier_env *env, int func_id) -++static bool check_raw_mode_ok(const struct bpf_func_proto *fn) -++{ -++ int count = 0; -++ -++ if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) -++ count++; -++ if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) -++ count++; -++ if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) -++ count++; -++ if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) -++ count++; -++ if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) -++ count++; -++ -++ /* We only support one arg being in raw mode at the moment, -++ * which is sufficient for the helper functions we have -++ * right now. -++ */ -++ return count <= 1; -++} -++ -++static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, -++ enum bpf_arg_type arg_next) -++{ -++ return (arg_type_is_mem_ptr(arg_curr) && -++ !arg_type_is_mem_size(arg_next)) || -++ (!arg_type_is_mem_ptr(arg_curr) && -++ arg_type_is_mem_size(arg_next)); -++} -++ -++static bool check_arg_pair_ok(const struct bpf_func_proto *fn) -++{ -++ /* bpf_xxx(..., buf, len) call will access 'len' -++ * bytes from memory 'buf'. Both arg types need -++ * to be paired, so make sure there's no buggy -++ * helper function specification. -++ */ -++ if (arg_type_is_mem_size(fn->arg1_type) || -++ arg_type_is_mem_ptr(fn->arg5_type) || -++ check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || -++ check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || -++ check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || -++ check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) -++ return false; -++ -++ return true; -++} -++ -++static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) -++{ -++ int count = 0; -++ -++ if (arg_type_may_be_refcounted(fn->arg1_type)) -++ count++; -++ if (arg_type_may_be_refcounted(fn->arg2_type)) -++ count++; -++ if (arg_type_may_be_refcounted(fn->arg3_type)) -++ count++; -++ if (arg_type_may_be_refcounted(fn->arg4_type)) -++ count++; -++ if (arg_type_may_be_refcounted(fn->arg5_type)) -++ count++; -++ -++ /* A reference acquiring function cannot acquire -++ * another refcounted ptr. -++ */ -++ if (is_acquire_function(func_id) && count) -++ return false; -++ -++ /* We only support one arg being unreferenced at the moment, -++ * which is sufficient for the helper functions we have right now. -++ */ -++ return count <= 1; -++} -++ -++static int check_func_proto(const struct bpf_func_proto *fn, int func_id) -++{ -++ return check_raw_mode_ok(fn) && -++ check_arg_pair_ok(fn) && -++ check_refcount_ok(fn, func_id) ? 0 : -EINVAL; -++} -++ -++/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] -++ * are now invalid, so turn them into unknown SCALAR_VALUE. -++ */ -++static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, -++ struct bpf_func_state *state) -++{ -++ struct bpf_reg_state *regs = state->regs, *reg; -++ int i; -++ -++ for (i = 0; i < MAX_BPF_REG; i++) -++ if (reg_is_pkt_pointer_any(®s[i])) -++ mark_reg_unknown(env, regs, i); -++ -++ bpf_for_each_spilled_reg(i, state, reg) { -++ if (!reg) -++ continue; -++ if (reg_is_pkt_pointer_any(reg)) -++ __mark_reg_unknown(env, reg); -++ } -++} -++ -++static void clear_all_pkt_pointers(struct bpf_verifier_env *env) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ int i; -++ -++ for (i = 0; i <= vstate->curframe; i++) -++ __clear_all_pkt_pointers(env, vstate->frame[i]); -++} -++ -++static void release_reg_references(struct bpf_verifier_env *env, -++ struct bpf_func_state *state, -++ int ref_obj_id) -++{ -++ struct bpf_reg_state *regs = state->regs, *reg; -++ int i; -++ -++ for (i = 0; i < MAX_BPF_REG; i++) -++ if (regs[i].ref_obj_id == ref_obj_id) -++ mark_reg_unknown(env, regs, i); -++ -++ bpf_for_each_spilled_reg(i, state, reg) { -++ if (!reg) -++ continue; -++ if (reg->ref_obj_id == ref_obj_id) -++ __mark_reg_unknown(env, reg); -++ } -++} -++ -++/* The pointer with the specified id has released its reference to kernel -++ * resources. Identify all copies of the same pointer and clear the reference. -++ */ -++static int release_reference(struct bpf_verifier_env *env, -++ int ref_obj_id) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ int err; -++ int i; -++ -++ err = release_reference_state(cur_func(env), ref_obj_id); -++ if (err) -++ return err; -++ -++ for (i = 0; i <= vstate->curframe; i++) -++ release_reg_references(env, vstate->frame[i], ref_obj_id); -++ -++ return 0; -++} -++ -++static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, -++ int *insn_idx) -++{ -++ struct bpf_verifier_state *state = env->cur_state; -++ struct bpf_func_state *caller, *callee; -++ int i, err, subprog, target_insn; -++ -++ if (state->curframe + 1 >= MAX_CALL_FRAMES) { -++ verbose(env, "the call stack of %d frames is too deep\n", -++ state->curframe + 2); -++ return -E2BIG; -++ } -++ -++ target_insn = *insn_idx + insn->imm; -++ subprog = find_subprog(env, target_insn + 1); -++ if (subprog < 0) { -++ verbose(env, "verifier bug. No program starts at insn %d\n", -++ target_insn + 1); -++ return -EFAULT; -++ } -++ -++ caller = state->frame[state->curframe]; -++ if (state->frame[state->curframe + 1]) { -++ verbose(env, "verifier bug. Frame %d already allocated\n", -++ state->curframe + 1); -++ return -EFAULT; -++ } -++ -++ callee = kzalloc(sizeof(*callee), GFP_KERNEL); -++ if (!callee) -++ return -ENOMEM; -++ state->frame[state->curframe + 1] = callee; -++ -++ /* callee cannot access r0, r6 - r9 for reading and has to write -++ * into its own stack before reading from it. -++ * callee can read/write into caller's stack -++ */ -++ init_func_state(env, callee, -++ /* remember the callsite, it will be used by bpf_exit */ -++ *insn_idx /* callsite */, -++ state->curframe + 1 /* frameno within this callchain */, -++ subprog /* subprog number within this prog */); -++ -++ /* Transfer references to the callee */ -++ err = transfer_reference_state(callee, caller); -++ if (err) -++ return err; -++ -++ /* copy r1 - r5 args that callee can access. The copy includes parent -++ * pointers, which connects us up to the liveness chain -++ */ -++ for (i = BPF_REG_1; i <= BPF_REG_5; i++) -++ callee->regs[i] = caller->regs[i]; -++ -++ /* after the call registers r0 - r5 were scratched */ -++ for (i = 0; i < CALLER_SAVED_REGS; i++) { -++ mark_reg_not_init(env, caller->regs, caller_saved[i]); -++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); -++ } -++ -++ /* only increment it after check_reg_arg() finished */ -++ state->curframe++; -++ -++ /* and go analyze first insn of the callee */ -++ *insn_idx = target_insn; -++ -++ if (env->log.level & BPF_LOG_LEVEL) { -++ verbose(env, "caller:\n"); -++ print_verifier_state(env, caller); -++ verbose(env, "callee:\n"); -++ print_verifier_state(env, callee); -++ } -++ return 0; -++} -++ -++static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) -++{ -++ struct bpf_verifier_state *state = env->cur_state; -++ struct bpf_func_state *caller, *callee; -++ struct bpf_reg_state *r0; -++ int err; -++ -++ callee = state->frame[state->curframe]; -++ r0 = &callee->regs[BPF_REG_0]; -++ if (r0->type == PTR_TO_STACK) { -++ /* technically it's ok to return caller's stack pointer -++ * (or caller's caller's pointer) back to the caller, -++ * since these pointers are valid. Only current stack -++ * pointer will be invalid as soon as function exits, -++ * but let's be conservative -++ */ -++ verbose(env, "cannot return stack pointer to the caller\n"); -++ return -EINVAL; -++ } -++ -++ state->curframe--; -++ caller = state->frame[state->curframe]; -++ /* return to the caller whatever r0 had in the callee */ -++ caller->regs[BPF_REG_0] = *r0; -++ -++ /* Transfer references to the caller */ -++ err = transfer_reference_state(caller, callee); -++ if (err) -++ return err; -++ -++ *insn_idx = callee->callsite + 1; -++ if (env->log.level & BPF_LOG_LEVEL) { -++ verbose(env, "returning from callee:\n"); -++ print_verifier_state(env, callee); -++ verbose(env, "to caller at %d:\n", *insn_idx); -++ print_verifier_state(env, caller); -++ } -++ /* clear everything in the callee */ -++ free_func_state(callee); -++ state->frame[state->curframe + 1] = NULL; -++ return 0; -++} -++ -++static int do_refine_retval_range(struct bpf_verifier_env *env, -++ struct bpf_reg_state *regs, int ret_type, -++ int func_id, struct bpf_call_arg_meta *meta) -++{ -++ struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; -++ struct bpf_reg_state tmp_reg = *ret_reg; -++ bool ret; -++ -++ if (ret_type != RET_INTEGER || -++ (func_id != BPF_FUNC_get_stack && -++ func_id != BPF_FUNC_probe_read_str)) -++ return 0; -++ -++ /* Error case where ret is in interval [S32MIN, -1]. */ -++ ret_reg->smin_value = S32_MIN; -++ ret_reg->smax_value = -1; -++ -++ __reg_deduce_bounds(ret_reg); -++ __reg_bound_offset(ret_reg); -++ __update_reg_bounds(ret_reg); -++ -++ ret = push_stack(env, env->insn_idx + 1, env->insn_idx, false); -++ if (!ret) -++ return -EFAULT; -++ -++ *ret_reg = tmp_reg; -++ -++ /* Success case where ret is in range [0, msize_max_value]. */ -++ ret_reg->smin_value = 0; -++ ret_reg->smax_value = meta->msize_max_value; -++ ret_reg->umin_value = ret_reg->smin_value; -++ ret_reg->umax_value = ret_reg->smax_value; -++ -++ __reg_deduce_bounds(ret_reg); -++ __reg_bound_offset(ret_reg); -++ __update_reg_bounds(ret_reg); -++ -++ return 0; -++} -++ -++static int -++record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, -++ int func_id, int insn_idx) -++{ -++ struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; -++ struct bpf_map *map = meta->map_ptr; -++ -++ if (func_id != BPF_FUNC_tail_call && -++ func_id != BPF_FUNC_map_lookup_elem && -++ func_id != BPF_FUNC_map_update_elem && -++ func_id != BPF_FUNC_map_delete_elem && -++ func_id != BPF_FUNC_map_push_elem && -++ func_id != BPF_FUNC_map_pop_elem && -++ func_id != BPF_FUNC_map_peek_elem) -++ return 0; -++ -++ if (map == NULL) { -++ verbose(env, "kernel subsystem misconfigured verifier\n"); -++ return -EINVAL; -++ } -++ -++ /* In case of read-only, some additional restrictions -++ * need to be applied in order to prevent altering the -++ * state of the map from program side. -++ */ -++ if ((map->map_flags & BPF_F_RDONLY_PROG) && -++ (func_id == BPF_FUNC_map_delete_elem || -++ func_id == BPF_FUNC_map_update_elem || -++ func_id == BPF_FUNC_map_push_elem || -++ func_id == BPF_FUNC_map_pop_elem)) { -++ verbose(env, "write into map forbidden\n"); -++ return -EACCES; -++ } -++ -++ if (!BPF_MAP_PTR(aux->map_state)) -++ bpf_map_ptr_store(aux, meta->map_ptr, -++ meta->map_ptr->unpriv_array); -++ else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr) -++ bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, -++ meta->map_ptr->unpriv_array); -++ return 0; -++} -++ -++static int check_reference_leak(struct bpf_verifier_env *env) -++{ -++ struct bpf_func_state *state = cur_func(env); -++ int i; -++ -++ for (i = 0; i < state->acquired_refs; i++) { -++ verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", -++ state->refs[i].id, state->refs[i].insn_idx); -++ } -++ return state->acquired_refs ? -EINVAL : 0; -++} -++ -++static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) -+ { -+- struct verifier_state *state = &env->cur_state; -+ const struct bpf_func_proto *fn = NULL; -+- struct reg_state *regs = state->regs; -+- struct bpf_map *map = NULL; -+- struct reg_state *reg; -++ struct bpf_reg_state *regs; -++ struct bpf_call_arg_meta meta; -++ bool changes_data; -+ int i, err; -+ -+ /* find function prototype */ -+ if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { -+- verbose("invalid func %d\n", func_id); -++ verbose(env, "invalid func %s#%d\n", func_id_name(func_id), -++ func_id); -+ return -EINVAL; -+ } -+ -+- if (env->prog->aux->ops->get_func_proto) -+- fn = env->prog->aux->ops->get_func_proto(func_id); -+- -++ if (env->ops->get_func_proto) -++ fn = env->ops->get_func_proto(func_id, env->prog); -+ if (!fn) { -+- verbose("unknown func %d\n", func_id); -++ verbose(env, "unknown func %s#%d\n", func_id_name(func_id), -++ func_id); -+ return -EINVAL; -+ } -+ -+ /* eBPF programs must be GPL compatible to use GPL-ed functions */ -+ if (!env->prog->gpl_compatible && fn->gpl_only) { -+- verbose("cannot call GPL only function from proprietary program\n"); -++ verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); -++ return -EINVAL; -++ } -++ -++ /* With LD_ABS/IND some JITs save/restore skb from r1. */ -++ changes_data = bpf_helper_changes_pkt_data(fn->func); -++ if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { -++ verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", -++ func_id_name(func_id), func_id); -+ return -EINVAL; -+ } -+ -++ memset(&meta, 0, sizeof(meta)); -++ meta.pkt_access = fn->pkt_access; -++ -++ err = check_func_proto(fn, func_id); -++ if (err) { -++ verbose(env, "kernel subsystem misconfigured func %s#%d\n", -++ func_id_name(func_id), func_id); -++ return err; -++ } -++ -++ meta.func_id = func_id; -+ /* check args */ -+- err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &map); -++ err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta); -+ if (err) -+ return err; -+- err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &map); -++ err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta); -+ if (err) -+ return err; -+- err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &map); -++ err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta); -+ if (err) -+ return err; -+- err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &map); -++ err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta); -+ if (err) -+ return err; -+- err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &map); -++ err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta); -+ if (err) -+ return err; -+ -++ err = record_func_map(env, &meta, func_id, insn_idx); -++ if (err) -++ return err; -++ -++ /* Mark slots with STACK_MISC in case of raw mode, stack offset -++ * is inferred from register state. -++ */ -++ for (i = 0; i < meta.access_size; i++) { -++ err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, -++ BPF_WRITE, -1, false); -++ if (err) -++ return err; -++ } -++ -++ if (func_id == BPF_FUNC_tail_call) { -++ err = check_reference_leak(env); -++ if (err) { -++ verbose(env, "tail_call would lead to reference leak\n"); -++ return err; -++ } -++ } else if (is_release_function(func_id)) { -++ err = release_reference(env, meta.ref_obj_id); -++ if (err) { -++ verbose(env, "func %s#%d reference has not been acquired before\n", -++ func_id_name(func_id), func_id); -++ return err; -++ } -++ } -++ -++ regs = cur_regs(env); -++ -++ /* check that flags argument in get_local_storage(map, flags) is 0, -++ * this is required because get_local_storage() can't return an error. -++ */ -++ if (func_id == BPF_FUNC_get_local_storage && -++ !register_is_null(®s[BPF_REG_2])) { -++ verbose(env, "get_local_storage() doesn't support non-zero flags\n"); -++ return -EINVAL; -++ } -++ -+ /* reset caller saved regs */ -+ for (i = 0; i < CALLER_SAVED_REGS; i++) { -+- reg = regs + caller_saved[i]; -+- reg->type = NOT_INIT; -+- reg->imm = 0; -++ mark_reg_not_init(env, regs, caller_saved[i]); -++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); -+ } -+ -+- /* update return register */ -++ /* helper call returns 64-bit value. */ -++ regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; -++ -++ /* update return register (already marked as written above) */ -+ if (fn->ret_type == RET_INTEGER) { -+- regs[BPF_REG_0].type = UNKNOWN_VALUE; -++ /* sets type to SCALAR_VALUE */ -++ mark_reg_unknown(env, regs, BPF_REG_0); -+ } else if (fn->ret_type == RET_VOID) { -+ regs[BPF_REG_0].type = NOT_INIT; -+- } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL) { -+- regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; -++ } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || -++ fn->ret_type == RET_PTR_TO_MAP_VALUE) { -++ /* There is no offset yet applied, variable or fixed */ -++ mark_reg_known_zero(env, regs, BPF_REG_0); -+ /* remember map_ptr, so that check_map_access() -+ * can check 'value_size' boundary of memory access -+ * to map element returned from bpf_map_lookup_elem() -+ */ -+- if (map == NULL) { -+- verbose("kernel subsystem misconfigured verifier\n"); -++ if (meta.map_ptr == NULL) { -++ verbose(env, -++ "kernel subsystem misconfigured verifier\n"); -+ return -EINVAL; -+ } -+- regs[BPF_REG_0].map_ptr = map; -++ regs[BPF_REG_0].map_ptr = meta.map_ptr; -++ if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { -++ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; -++ if (map_value_has_spin_lock(meta.map_ptr)) -++ regs[BPF_REG_0].id = ++env->id_gen; -++ } else { -++ regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; -++ regs[BPF_REG_0].id = ++env->id_gen; -++ } -++ } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { -++ mark_reg_known_zero(env, regs, BPF_REG_0); -++ regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; -++ regs[BPF_REG_0].id = ++env->id_gen; -++ } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { -++ mark_reg_known_zero(env, regs, BPF_REG_0); -++ regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; -++ regs[BPF_REG_0].id = ++env->id_gen; -++ } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { -++ mark_reg_known_zero(env, regs, BPF_REG_0); -++ regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; -++ regs[BPF_REG_0].id = ++env->id_gen; -+ } else { -+- verbose("unknown return type %d of func %d\n", -+- fn->ret_type, func_id); -++ verbose(env, "unknown return type %d of func %s#%d\n", -++ fn->ret_type, func_id_name(func_id), func_id); -+ return -EINVAL; -+ } -+ -+- err = check_map_func_compatibility(map, func_id); -++ if (is_ptr_cast_function(func_id)) { -++ /* For release_reference() */ -++ regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; -++ } else if (is_acquire_function(func_id)) { -++ int id = acquire_reference_state(env, insn_idx); -++ -++ if (id < 0) -++ return id; -++ /* For mark_ptr_or_null_reg() */ -++ regs[BPF_REG_0].id = id; -++ /* For release_reference() */ -++ regs[BPF_REG_0].ref_obj_id = id; -++ } -++ -++ err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); -++ if (err) -++ return err; -++ -++ err = check_map_func_compatibility(env, meta.map_ptr, func_id); -+ if (err) -+ return err; -+ -++ if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) { -++ const char *err_str; -++ -++#ifdef CONFIG_PERF_EVENTS -++ err = get_callchain_buffers(sysctl_perf_event_max_stack); -++ err_str = "cannot get callchain buffer for func %s#%d\n"; -++#else -++ err = -ENOTSUPP; -++ err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; -++#endif -++ if (err) { -++ verbose(env, err_str, func_id_name(func_id), func_id); -++ return err; -++ } -++ -++ env->prog->has_callchain_buf = true; -++ } -++ -++ if (changes_data) -++ clear_all_pkt_pointers(env); -+ return 0; -+ } -+ -++static bool signed_add_overflows(s64 a, s64 b) -++{ -++ /* Do the add in u64, where overflow is well-defined */ -++ s64 res = (s64)((u64)a + (u64)b); -++ -++ if (b < 0) -++ return res > a; -++ return res < a; -++} -++ -++static bool signed_sub_overflows(s64 a, s64 b) -++{ -++ /* Do the sub in u64, where overflow is well-defined */ -++ s64 res = (s64)((u64)a - (u64)b); -++ -++ if (b < 0) -++ return res < a; -++ return res > a; -++} -++ -++static bool check_reg_sane_offset(struct bpf_verifier_env *env, -++ const struct bpf_reg_state *reg, -++ enum bpf_reg_type type) -++{ -++ bool known = tnum_is_const(reg->var_off); -++ s64 val = reg->var_off.value; -++ s64 smin = reg->smin_value; -++ -++ if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { -++ verbose(env, "math between %s pointer and %lld is not allowed\n", -++ reg_type_str[type], val); -++ return false; -++ } -++ -++ if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { -++ verbose(env, "%s pointer offset %d is not allowed\n", -++ reg_type_str[type], reg->off); -++ return false; -++ } -++ -++ if (smin == S64_MIN) { -++ verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", -++ reg_type_str[type]); -++ return false; -++ } -++ -++ if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { -++ verbose(env, "value %lld makes %s pointer be out of bounds\n", -++ smin, reg_type_str[type]); -++ return false; -++ } -++ -++ return true; -++} -++ -++static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) -++{ -++ return &env->insn_aux_data[env->insn_idx]; -++} -++ -++enum { -++ REASON_BOUNDS = -1, -++ REASON_TYPE = -2, -++ REASON_PATHS = -3, -++ REASON_LIMIT = -4, -++ REASON_STACK = -5, -++}; -++ -++static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, -++ u32 *alu_limit, bool mask_to_left) -++{ -++ u32 max = 0, ptr_limit = 0; -++ -++ switch (ptr_reg->type) { -++ case PTR_TO_STACK: -++ /* Offset 0 is out-of-bounds, but acceptable start for the -++ * left direction, see BPF_REG_FP. Also, unknown scalar -++ * offset where we would need to deal with min/max bounds is -++ * currently prohibited for unprivileged. -++ */ -++ max = MAX_BPF_STACK + mask_to_left; -++ ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off); -++ break; -++ case PTR_TO_MAP_VALUE: -++ max = ptr_reg->map_ptr->value_size; -++ ptr_limit = (mask_to_left ? -++ ptr_reg->smin_value : -++ ptr_reg->umax_value) + ptr_reg->off; -++ break; -++ default: -++ return REASON_TYPE; -++ } -++ -++ if (ptr_limit >= max) -++ return REASON_LIMIT; -++ *alu_limit = ptr_limit; -++ return 0; -++} -++ -++static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, -++ const struct bpf_insn *insn) -++{ -++ return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K; -++} -++ -++static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, -++ u32 alu_state, u32 alu_limit) -++{ -++ /* If we arrived here from different branches with different -++ * state or limits to sanitize, then this won't work. -++ */ -++ if (aux->alu_state && -++ (aux->alu_state != alu_state || -++ aux->alu_limit != alu_limit)) -++ return REASON_PATHS; -++ -++ /* Corresponding fixup done in fixup_bpf_calls(). */ -++ aux->alu_state = alu_state; -++ aux->alu_limit = alu_limit; -++ return 0; -++} -++ -++static int sanitize_val_alu(struct bpf_verifier_env *env, -++ struct bpf_insn *insn) -++{ -++ struct bpf_insn_aux_data *aux = cur_aux(env); -++ -++ if (can_skip_alu_sanitation(env, insn)) -++ return 0; -++ -++ return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); -++} -++ -++static bool sanitize_needed(u8 opcode) -++{ -++ return opcode == BPF_ADD || opcode == BPF_SUB; -++} -++ -++struct bpf_sanitize_info { -++ struct bpf_insn_aux_data aux; -++ bool mask_to_left; -++}; -++ -++static struct bpf_verifier_state * -++sanitize_speculative_path(struct bpf_verifier_env *env, -++ const struct bpf_insn *insn, -++ u32 next_idx, u32 curr_idx) -++{ -++ struct bpf_verifier_state *branch; -++ struct bpf_reg_state *regs; -++ -++ branch = push_stack(env, next_idx, curr_idx, true); -++ if (branch && insn) { -++ regs = branch->frame[branch->curframe]->regs; -++ if (BPF_SRC(insn->code) == BPF_K) { -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ } else if (BPF_SRC(insn->code) == BPF_X) { -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ mark_reg_unknown(env, regs, insn->src_reg); -++ } -++ } -++ return branch; -++} -++ -++static int sanitize_ptr_alu(struct bpf_verifier_env *env, -++ struct bpf_insn *insn, -++ const struct bpf_reg_state *ptr_reg, -++ const struct bpf_reg_state *off_reg, -++ struct bpf_reg_state *dst_reg, -++ struct bpf_sanitize_info *info, -++ const bool commit_window) -++{ -++ struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; -++ struct bpf_verifier_state *vstate = env->cur_state; -++ bool off_is_imm = tnum_is_const(off_reg->var_off); -++ bool off_is_neg = off_reg->smin_value < 0; -++ bool ptr_is_dst_reg = ptr_reg == dst_reg; -++ u8 opcode = BPF_OP(insn->code); -++ u32 alu_state, alu_limit; -++ struct bpf_reg_state tmp; -++ bool ret; -++ int err; -++ -++ if (can_skip_alu_sanitation(env, insn)) -++ return 0; -++ -++ /* We already marked aux for masking from non-speculative -++ * paths, thus we got here in the first place. We only care -++ * to explore bad access from here. -++ */ -++ if (vstate->speculative) -++ goto do_sim; -++ -++ if (!commit_window) { -++ if (!tnum_is_const(off_reg->var_off) && -++ (off_reg->smin_value < 0) != (off_reg->smax_value < 0)) -++ return REASON_BOUNDS; -++ -++ info->mask_to_left = (opcode == BPF_ADD && off_is_neg) || -++ (opcode == BPF_SUB && !off_is_neg); -++ } -++ -++ err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left); -++ if (err < 0) -++ return err; -++ -++ if (commit_window) { -++ /* In commit phase we narrow the masking window based on -++ * the observed pointer move after the simulated operation. -++ */ -++ alu_state = info->aux.alu_state; -++ alu_limit = abs(info->aux.alu_limit - alu_limit); -++ } else { -++ alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; -++ alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0; -++ alu_state |= ptr_is_dst_reg ? -++ BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; -++ } -++ -++ err = update_alu_sanitation_state(aux, alu_state, alu_limit); -++ if (err < 0) -++ return err; -++do_sim: -++ /* If we're in commit phase, we're done here given we already -++ * pushed the truncated dst_reg into the speculative verification -++ * stack. -++ * -++ * Also, when register is a known constant, we rewrite register-based -++ * operation to immediate-based, and thus do not need masking (and as -++ * a consequence, do not need to simulate the zero-truncation either). -++ */ -++ if (commit_window || off_is_imm) -++ return 0; -++ -++ /* Simulate and find potential out-of-bounds access under -++ * speculative execution from truncation as a result of -++ * masking when off was not within expected range. If off -++ * sits in dst, then we temporarily need to move ptr there -++ * to simulate dst (== 0) +/-= ptr. Needed, for example, -++ * for cases where we use K-based arithmetic in one direction -++ * and truncated reg-based in the other in order to explore -++ * bad access. -++ */ -++ if (!ptr_is_dst_reg) { -++ tmp = *dst_reg; -++ *dst_reg = *ptr_reg; -++ } -++ ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, -++ env->insn_idx); -++ if (!ptr_is_dst_reg && ret) -++ *dst_reg = tmp; -++ return !ret ? REASON_STACK : 0; -++} -++ -++static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ -++ /* If we simulate paths under speculation, we don't update the -++ * insn as 'seen' such that when we verify unreachable paths in -++ * the non-speculative domain, sanitize_dead_code() can still -++ * rewrite/sanitize them. -++ */ -++ if (!vstate->speculative) -++ env->insn_aux_data[env->insn_idx].seen = true; -++} -++ -++static int sanitize_err(struct bpf_verifier_env *env, -++ const struct bpf_insn *insn, int reason, -++ const struct bpf_reg_state *off_reg, -++ const struct bpf_reg_state *dst_reg) -++{ -++ static const char *err = "pointer arithmetic with it prohibited for !root"; -++ const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; -++ u32 dst = insn->dst_reg, src = insn->src_reg; -++ -++ switch (reason) { -++ case REASON_BOUNDS: -++ verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", -++ off_reg == dst_reg ? dst : src, err); -++ break; -++ case REASON_TYPE: -++ verbose(env, "R%d has pointer with unsupported alu operation, %s\n", -++ off_reg == dst_reg ? src : dst, err); -++ break; -++ case REASON_PATHS: -++ verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", -++ dst, op, err); -++ break; -++ case REASON_LIMIT: -++ verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", -++ dst, op, err); -++ break; -++ case REASON_STACK: -++ verbose(env, "R%d could not be pushed for speculative verification, %s\n", -++ dst, err); -++ break; -++ default: -++ verbose(env, "verifier internal error: unknown reason (%d)\n", -++ reason); -++ break; -++ } -++ -++ return -EACCES; -++} -++ -++static int sanitize_check_bounds(struct bpf_verifier_env *env, -++ const struct bpf_insn *insn, -++ const struct bpf_reg_state *dst_reg) -++{ -++ u32 dst = insn->dst_reg; -++ -++ /* For unprivileged we require that resulting offset must be in bounds -++ * in order to be able to sanitize access later on. -++ */ -++ if (env->allow_ptr_leaks) -++ return 0; -++ -++ switch (dst_reg->type) { -++ case PTR_TO_STACK: -++ if (check_stack_access(env, dst_reg, dst_reg->off + -++ dst_reg->var_off.value, 1)) { -++ verbose(env, "R%d stack pointer arithmetic goes out of range, " -++ "prohibited for !root\n", dst); -++ return -EACCES; -++ } -++ break; -++ case PTR_TO_MAP_VALUE: -++ if (check_map_access(env, dst, dst_reg->off, 1, false)) { -++ verbose(env, "R%d pointer arithmetic of map value goes out of range, " -++ "prohibited for !root\n", dst); -++ return -EACCES; -++ } -++ break; -++ default: -++ break; -++ } -++ -++ return 0; -++} -++ -++/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. -++ * Caller should also handle BPF_MOV case separately. -++ * If we return -EACCES, caller may want to try again treating pointer as a -++ * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. -++ */ -++static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, -++ struct bpf_insn *insn, -++ const struct bpf_reg_state *ptr_reg, -++ const struct bpf_reg_state *off_reg) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ struct bpf_reg_state *regs = state->regs, *dst_reg; -++ bool known = tnum_is_const(off_reg->var_off); -++ s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, -++ smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; -++ u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, -++ umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; -++ struct bpf_sanitize_info info = {}; -++ u8 opcode = BPF_OP(insn->code); -++ u32 dst = insn->dst_reg; -++ int ret; -++ -++ dst_reg = ®s[dst]; -++ -++ if ((known && (smin_val != smax_val || umin_val != umax_val)) || -++ smin_val > smax_val || umin_val > umax_val) { -++ /* Taint dst register if offset had invalid bounds derived from -++ * e.g. dead branches. -++ */ -++ __mark_reg_unknown(env, dst_reg); -++ return 0; -++ } -++ -++ if (BPF_CLASS(insn->code) != BPF_ALU64) { -++ /* 32-bit ALU ops on pointers produce (meaningless) scalars */ -++ verbose(env, -++ "R%d 32-bit pointer arithmetic prohibited\n", -++ dst); -++ return -EACCES; -++ } -++ -++ switch (ptr_reg->type) { -++ case PTR_TO_MAP_VALUE_OR_NULL: -++ verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", -++ dst, reg_type_str[ptr_reg->type]); -++ return -EACCES; -++ case CONST_PTR_TO_MAP: -++ /* smin_val represents the known value */ -++ if (known && smin_val == 0 && opcode == BPF_ADD) -++ break; -++ /* fall-through */ -++ case PTR_TO_PACKET_END: -++ case PTR_TO_SOCKET: -++ case PTR_TO_SOCKET_OR_NULL: -++ case PTR_TO_SOCK_COMMON: -++ case PTR_TO_SOCK_COMMON_OR_NULL: -++ case PTR_TO_TCP_SOCK: -++ case PTR_TO_TCP_SOCK_OR_NULL: -++ case PTR_TO_XDP_SOCK: -++ verbose(env, "R%d pointer arithmetic on %s prohibited\n", -++ dst, reg_type_str[ptr_reg->type]); -++ return -EACCES; -++ default: -++ break; -++ } -++ -++ /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. -++ * The id may be overwritten later if we create a new variable offset. -++ */ -++ dst_reg->type = ptr_reg->type; -++ dst_reg->id = ptr_reg->id; -++ -++ if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || -++ !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) -++ return -EINVAL; -++ -++ if (sanitize_needed(opcode)) { -++ ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, -++ &info, false); -++ if (ret < 0) -++ return sanitize_err(env, insn, ret, off_reg, dst_reg); -++ } -++ -++ switch (opcode) { -++ case BPF_ADD: -++ /* We can take a fixed offset as long as it doesn't overflow -++ * the s32 'off' field -++ */ -++ if (known && (ptr_reg->off + smin_val == -++ (s64)(s32)(ptr_reg->off + smin_val))) { -++ /* pointer += K. Accumulate it into fixed offset */ -++ dst_reg->smin_value = smin_ptr; -++ dst_reg->smax_value = smax_ptr; -++ dst_reg->umin_value = umin_ptr; -++ dst_reg->umax_value = umax_ptr; -++ dst_reg->var_off = ptr_reg->var_off; -++ dst_reg->off = ptr_reg->off + smin_val; -++ dst_reg->raw = ptr_reg->raw; -++ break; -++ } -++ /* A new variable offset is created. Note that off_reg->off -++ * == 0, since it's a scalar. -++ * dst_reg gets the pointer type and since some positive -++ * integer value was added to the pointer, give it a new 'id' -++ * if it's a PTR_TO_PACKET. -++ * this creates a new 'base' pointer, off_reg (variable) gets -++ * added into the variable offset, and we copy the fixed offset -++ * from ptr_reg. -++ */ -++ if (signed_add_overflows(smin_ptr, smin_val) || -++ signed_add_overflows(smax_ptr, smax_val)) { -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ dst_reg->smin_value = smin_ptr + smin_val; -++ dst_reg->smax_value = smax_ptr + smax_val; -++ } -++ if (umin_ptr + umin_val < umin_ptr || -++ umax_ptr + umax_val < umax_ptr) { -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ } else { -++ dst_reg->umin_value = umin_ptr + umin_val; -++ dst_reg->umax_value = umax_ptr + umax_val; -++ } -++ dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); -++ dst_reg->off = ptr_reg->off; -++ dst_reg->raw = ptr_reg->raw; -++ if (reg_is_pkt_pointer(ptr_reg)) { -++ dst_reg->id = ++env->id_gen; -++ /* something was added to pkt_ptr, set range to zero */ -++ dst_reg->raw = 0; -++ } -++ break; -++ case BPF_SUB: -++ if (dst_reg == off_reg) { -++ /* scalar -= pointer. Creates an unknown scalar */ -++ verbose(env, "R%d tried to subtract pointer from scalar\n", -++ dst); -++ return -EACCES; -++ } -++ /* We don't allow subtraction from FP, because (according to -++ * test_verifier.c test "invalid fp arithmetic", JITs might not -++ * be able to deal with it. -++ */ -++ if (ptr_reg->type == PTR_TO_STACK) { -++ verbose(env, "R%d subtraction from stack pointer prohibited\n", -++ dst); -++ return -EACCES; -++ } -++ if (known && (ptr_reg->off - smin_val == -++ (s64)(s32)(ptr_reg->off - smin_val))) { -++ /* pointer -= K. Subtract it from fixed offset */ -++ dst_reg->smin_value = smin_ptr; -++ dst_reg->smax_value = smax_ptr; -++ dst_reg->umin_value = umin_ptr; -++ dst_reg->umax_value = umax_ptr; -++ dst_reg->var_off = ptr_reg->var_off; -++ dst_reg->id = ptr_reg->id; -++ dst_reg->off = ptr_reg->off - smin_val; -++ dst_reg->raw = ptr_reg->raw; -++ break; -++ } -++ /* A new variable offset is created. If the subtrahend is known -++ * nonnegative, then any reg->range we had before is still good. -++ */ -++ if (signed_sub_overflows(smin_ptr, smax_val) || -++ signed_sub_overflows(smax_ptr, smin_val)) { -++ /* Overflow possible, we know nothing */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ dst_reg->smin_value = smin_ptr - smax_val; -++ dst_reg->smax_value = smax_ptr - smin_val; -++ } -++ if (umin_ptr < umax_val) { -++ /* Overflow possible, we know nothing */ -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ } else { -++ /* Cannot overflow (as long as bounds are consistent) */ -++ dst_reg->umin_value = umin_ptr - umax_val; -++ dst_reg->umax_value = umax_ptr - umin_val; -++ } -++ dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); -++ dst_reg->off = ptr_reg->off; -++ dst_reg->raw = ptr_reg->raw; -++ if (reg_is_pkt_pointer(ptr_reg)) { -++ dst_reg->id = ++env->id_gen; -++ /* something was added to pkt_ptr, set range to zero */ -++ if (smin_val < 0) -++ dst_reg->raw = 0; -++ } -++ break; -++ case BPF_AND: -++ case BPF_OR: -++ case BPF_XOR: -++ /* bitwise ops on pointers are troublesome, prohibit. */ -++ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", -++ dst, bpf_alu_string[opcode >> 4]); -++ return -EACCES; -++ default: -++ /* other operators (e.g. MUL,LSH) produce non-pointer results */ -++ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", -++ dst, bpf_alu_string[opcode >> 4]); -++ return -EACCES; -++ } -++ -++ if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) -++ return -EINVAL; -++ -++ __update_reg_bounds(dst_reg); -++ __reg_deduce_bounds(dst_reg); -++ __reg_bound_offset(dst_reg); -++ -++ if (sanitize_check_bounds(env, insn, dst_reg) < 0) -++ return -EACCES; -++ if (sanitize_needed(opcode)) { -++ ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, -++ &info, true); -++ if (ret < 0) -++ return sanitize_err(env, insn, ret, off_reg, dst_reg); -++ } -++ -++ return 0; -++} -++ -++/* WARNING: This function does calculations on 64-bit values, but the actual -++ * execution may occur on 32-bit values. Therefore, things like bitshifts -++ * need extra checks in the 32-bit case. -++ */ -++static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, -++ struct bpf_insn *insn, -++ struct bpf_reg_state *dst_reg, -++ struct bpf_reg_state src_reg) -++{ -++ struct bpf_reg_state *regs = cur_regs(env); -++ u8 opcode = BPF_OP(insn->code); -++ bool src_known, dst_known; -++ s64 smin_val, smax_val; -++ u64 umin_val, umax_val; -++ u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; -++ int ret; -++ -++ if (insn_bitness == 32) { -++ /* Relevant for 32-bit RSH: Information can propagate towards -++ * LSB, so it isn't sufficient to only truncate the output to -++ * 32 bits. -++ */ -++ coerce_reg_to_size(dst_reg, 4); -++ coerce_reg_to_size(&src_reg, 4); -++ } -++ -++ smin_val = src_reg.smin_value; -++ smax_val = src_reg.smax_value; -++ umin_val = src_reg.umin_value; -++ umax_val = src_reg.umax_value; -++ src_known = tnum_is_const(src_reg.var_off); -++ dst_known = tnum_is_const(dst_reg->var_off); -++ -++ if ((src_known && (smin_val != smax_val || umin_val != umax_val)) || -++ smin_val > smax_val || umin_val > umax_val) { -++ /* Taint dst register if offset had invalid bounds derived from -++ * e.g. dead branches. -++ */ -++ __mark_reg_unknown(env, dst_reg); -++ return 0; -++ } -++ -++ if (!src_known && -++ opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { -++ __mark_reg_unknown(env, dst_reg); -++ return 0; -++ } -++ -++ if (sanitize_needed(opcode)) { -++ ret = sanitize_val_alu(env, insn); -++ if (ret < 0) -++ return sanitize_err(env, insn, ret, NULL, NULL); -++ } -++ -++ switch (opcode) { -++ case BPF_ADD: -++ if (signed_add_overflows(dst_reg->smin_value, smin_val) || -++ signed_add_overflows(dst_reg->smax_value, smax_val)) { -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ dst_reg->smin_value += smin_val; -++ dst_reg->smax_value += smax_val; -++ } -++ if (dst_reg->umin_value + umin_val < umin_val || -++ dst_reg->umax_value + umax_val < umax_val) { -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ } else { -++ dst_reg->umin_value += umin_val; -++ dst_reg->umax_value += umax_val; -++ } -++ dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); -++ break; -++ case BPF_SUB: -++ if (signed_sub_overflows(dst_reg->smin_value, smax_val) || -++ signed_sub_overflows(dst_reg->smax_value, smin_val)) { -++ /* Overflow possible, we know nothing */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ dst_reg->smin_value -= smax_val; -++ dst_reg->smax_value -= smin_val; -++ } -++ if (dst_reg->umin_value < umax_val) { -++ /* Overflow possible, we know nothing */ -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ } else { -++ /* Cannot overflow (as long as bounds are consistent) */ -++ dst_reg->umin_value -= umax_val; -++ dst_reg->umax_value -= umin_val; -++ } -++ dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); -++ break; -++ case BPF_MUL: -++ dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); -++ if (smin_val < 0 || dst_reg->smin_value < 0) { -++ /* Ain't nobody got time to multiply that sign */ -++ __mark_reg_unbounded(dst_reg); -++ __update_reg_bounds(dst_reg); -++ break; -++ } -++ /* Both values are positive, so we can work with unsigned and -++ * copy the result to signed (unless it exceeds S64_MAX). -++ */ -++ if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { -++ /* Potential overflow, we know nothing */ -++ __mark_reg_unbounded(dst_reg); -++ /* (except what we can learn from the var_off) */ -++ __update_reg_bounds(dst_reg); -++ break; -++ } -++ dst_reg->umin_value *= umin_val; -++ dst_reg->umax_value *= umax_val; -++ if (dst_reg->umax_value > S64_MAX) { -++ /* Overflow possible, we know nothing */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ dst_reg->smin_value = dst_reg->umin_value; -++ dst_reg->smax_value = dst_reg->umax_value; -++ } -++ break; -++ case BPF_AND: -++ if (src_known && dst_known) { -++ __mark_reg_known(dst_reg, dst_reg->var_off.value & -++ src_reg.var_off.value); -++ break; -++ } -++ /* We get our minimum from the var_off, since that's inherently -++ * bitwise. Our maximum is the minimum of the operands' maxima. -++ */ -++ dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); -++ dst_reg->umin_value = dst_reg->var_off.value; -++ dst_reg->umax_value = min(dst_reg->umax_value, umax_val); -++ if (dst_reg->smin_value < 0 || smin_val < 0) { -++ /* Lose signed bounds when ANDing negative numbers, -++ * ain't nobody got time for that. -++ */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ /* ANDing two positives gives a positive, so safe to -++ * cast result into s64. -++ */ -++ dst_reg->smin_value = dst_reg->umin_value; -++ dst_reg->smax_value = dst_reg->umax_value; -++ } -++ /* We may learn something more from the var_off */ -++ __update_reg_bounds(dst_reg); -++ break; -++ case BPF_OR: -++ if (src_known && dst_known) { -++ __mark_reg_known(dst_reg, dst_reg->var_off.value | -++ src_reg.var_off.value); -++ break; -++ } -++ /* We get our maximum from the var_off, and our minimum is the -++ * maximum of the operands' minima -++ */ -++ dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); -++ dst_reg->umin_value = max(dst_reg->umin_value, umin_val); -++ dst_reg->umax_value = dst_reg->var_off.value | -++ dst_reg->var_off.mask; -++ if (dst_reg->smin_value < 0 || smin_val < 0) { -++ /* Lose signed bounds when ORing negative numbers, -++ * ain't nobody got time for that. -++ */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ } else { -++ /* ORing two positives gives a positive, so safe to -++ * cast result into s64. -++ */ -++ dst_reg->smin_value = dst_reg->umin_value; -++ dst_reg->smax_value = dst_reg->umax_value; -++ } -++ /* We may learn something more from the var_off */ -++ __update_reg_bounds(dst_reg); -++ break; -++ case BPF_LSH: -++ if (umax_val >= insn_bitness) { -++ /* Shifts greater than 31 or 63 are undefined. -++ * This includes shifts by a negative number. -++ */ -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ break; -++ } -++ /* We lose all sign bit information (except what we can pick -++ * up from var_off) -++ */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ /* If we might shift our top bit out, then we know nothing */ -++ if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ } else { -++ dst_reg->umin_value <<= umin_val; -++ dst_reg->umax_value <<= umax_val; -++ } -++ dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); -++ /* We may learn something more from the var_off */ -++ __update_reg_bounds(dst_reg); -++ break; -++ case BPF_RSH: -++ if (umax_val >= insn_bitness) { -++ /* Shifts greater than 31 or 63 are undefined. -++ * This includes shifts by a negative number. -++ */ -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ break; -++ } -++ /* BPF_RSH is an unsigned shift. If the value in dst_reg might -++ * be negative, then either: -++ * 1) src_reg might be zero, so the sign bit of the result is -++ * unknown, so we lose our signed bounds -++ * 2) it's known negative, thus the unsigned bounds capture the -++ * signed bounds -++ * 3) the signed bounds cross zero, so they tell us nothing -++ * about the result -++ * If the value in dst_reg is known nonnegative, then again the -++ * unsigned bounts capture the signed bounds. -++ * Thus, in all cases it suffices to blow away our signed bounds -++ * and rely on inferring new ones from the unsigned bounds and -++ * var_off of the result. -++ */ -++ dst_reg->smin_value = S64_MIN; -++ dst_reg->smax_value = S64_MAX; -++ dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); -++ dst_reg->umin_value >>= umax_val; -++ dst_reg->umax_value >>= umin_val; -++ /* We may learn something more from the var_off */ -++ __update_reg_bounds(dst_reg); -++ break; -++ case BPF_ARSH: -++ if (umax_val >= insn_bitness) { -++ /* Shifts greater than 31 or 63 are undefined. -++ * This includes shifts by a negative number. -++ */ -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ break; -++ } -++ -++ /* Upon reaching here, src_known is true and -++ * umax_val is equal to umin_val. -++ */ -++ if (insn_bitness == 32) { -++ dst_reg->smin_value = (u32)(((s32)dst_reg->smin_value) >> umin_val); -++ dst_reg->smax_value = (u32)(((s32)dst_reg->smax_value) >> umin_val); -++ } else { -++ dst_reg->smin_value >>= umin_val; -++ dst_reg->smax_value >>= umin_val; -++ } -++ -++ dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, -++ insn_bitness); -++ -++ /* blow away the dst_reg umin_value/umax_value and rely on -++ * dst_reg var_off to refine the result. -++ */ -++ dst_reg->umin_value = 0; -++ dst_reg->umax_value = U64_MAX; -++ __update_reg_bounds(dst_reg); -++ break; -++ default: -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ break; -++ } -++ -++ if (BPF_CLASS(insn->code) != BPF_ALU64) { -++ /* 32-bit ALU ops are (32,32)->32 */ -++ coerce_reg_to_size(dst_reg, 4); -++ } -++ -++ __reg_deduce_bounds(dst_reg); -++ __reg_bound_offset(dst_reg); -++ return 0; -++} -++ -++/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max -++ * and var_off. -++ */ -++static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, -++ struct bpf_insn *insn) -++{ -++ struct bpf_verifier_state *vstate = env->cur_state; -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; -++ struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; -++ u8 opcode = BPF_OP(insn->code); -++ int err; -++ -++ dst_reg = ®s[insn->dst_reg]; -++ src_reg = NULL; -++ if (dst_reg->type != SCALAR_VALUE) -++ ptr_reg = dst_reg; -++ if (BPF_SRC(insn->code) == BPF_X) { -++ src_reg = ®s[insn->src_reg]; -++ if (src_reg->type != SCALAR_VALUE) { -++ if (dst_reg->type != SCALAR_VALUE) { -++ /* Combining two pointers by any ALU op yields -++ * an arbitrary scalar. Disallow all math except -++ * pointer subtraction -++ */ -++ if (opcode == BPF_SUB && env->allow_ptr_leaks) { -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ return 0; -++ } -++ verbose(env, "R%d pointer %s pointer prohibited\n", -++ insn->dst_reg, -++ bpf_alu_string[opcode >> 4]); -++ return -EACCES; -++ } else { -++ /* scalar += pointer -++ * This is legal, but we have to reverse our -++ * src/dest handling in computing the range -++ */ -++ err = mark_chain_precision(env, insn->dst_reg); -++ if (err) -++ return err; -++ return adjust_ptr_min_max_vals(env, insn, -++ src_reg, dst_reg); -++ } -++ } else if (ptr_reg) { -++ /* pointer += scalar */ -++ err = mark_chain_precision(env, insn->src_reg); -++ if (err) -++ return err; -++ return adjust_ptr_min_max_vals(env, insn, -++ dst_reg, src_reg); -++ } -++ } else { -++ /* Pretend the src is a reg with a known value, since we only -++ * need to be able to read from this state. -++ */ -++ off_reg.type = SCALAR_VALUE; -++ __mark_reg_known(&off_reg, insn->imm); -++ src_reg = &off_reg; -++ if (ptr_reg) /* pointer += K */ -++ return adjust_ptr_min_max_vals(env, insn, -++ ptr_reg, src_reg); -++ } -++ -++ /* Got here implies adding two SCALAR_VALUEs */ -++ if (WARN_ON_ONCE(ptr_reg)) { -++ print_verifier_state(env, state); -++ verbose(env, "verifier internal error: unexpected ptr_reg\n"); -++ return -EINVAL; -++ } -++ if (WARN_ON(!src_reg)) { -++ print_verifier_state(env, state); -++ verbose(env, "verifier internal error: no src_reg\n"); -++ return -EINVAL; -++ } -++ return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); -++} -++ -+ /* check validity of 32-bit and 64-bit arithmetic operations */ -+-static int check_alu_op(struct verifier_env *env, struct bpf_insn *insn) -++static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) -+ { -+- struct reg_state *regs = env->cur_state.regs; -++ struct bpf_reg_state *regs = cur_regs(env); -+ u8 opcode = BPF_OP(insn->code); -+ int err; -+ -+@@ -1034,30 +5134,31 @@ static int check_alu_op(struct verifier_ -+ if (BPF_SRC(insn->code) != 0 || -+ insn->src_reg != BPF_REG_0 || -+ insn->off != 0 || insn->imm != 0) { -+- verbose("BPF_NEG uses reserved fields\n"); -++ verbose(env, "BPF_NEG uses reserved fields\n"); -+ return -EINVAL; -+ } -+ } else { -+ if (insn->src_reg != BPF_REG_0 || insn->off != 0 || -+- (insn->imm != 16 && insn->imm != 32 && insn->imm != 64)) { -+- verbose("BPF_END uses reserved fields\n"); -++ (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || -++ BPF_CLASS(insn->code) == BPF_ALU64) { -++ verbose(env, "BPF_END uses reserved fields\n"); -+ return -EINVAL; -+ } -+ } -+ -+ /* check src operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -+ if (is_pointer_value(env, insn->dst_reg)) { -+- verbose("R%d pointer arithmetic prohibited\n", -++ verbose(env, "R%d pointer arithmetic prohibited\n", -+ insn->dst_reg); -+ return -EACCES; -+ } -+ -+ /* check dest operand */ -+- err = check_reg_arg(regs, insn->dst_reg, DST_OP); -++ err = check_reg_arg(env, insn->dst_reg, DST_OP); -+ if (err) -+ return err; -+ -+@@ -1065,81 +5166,100 @@ static int check_alu_op(struct verifier_ -+ -+ if (BPF_SRC(insn->code) == BPF_X) { -+ if (insn->imm != 0 || insn->off != 0) { -+- verbose("BPF_MOV uses reserved fields\n"); -++ verbose(env, "BPF_MOV uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ /* check src operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ } else { -+ if (insn->src_reg != BPF_REG_0 || insn->off != 0) { -+- verbose("BPF_MOV uses reserved fields\n"); -++ verbose(env, "BPF_MOV uses reserved fields\n"); -+ return -EINVAL; -+ } -+ } -+ -+- /* check dest operand */ -+- err = check_reg_arg(regs, insn->dst_reg, DST_OP); -++ /* check dest operand, mark as required later */ -++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); -+ if (err) -+ return err; -+ -+ if (BPF_SRC(insn->code) == BPF_X) { -++ struct bpf_reg_state *src_reg = regs + insn->src_reg; -++ struct bpf_reg_state *dst_reg = regs + insn->dst_reg; -++ -+ if (BPF_CLASS(insn->code) == BPF_ALU64) { -+ /* case: R1 = R2 -+ * copy register state to dest reg -+ */ -+- regs[insn->dst_reg] = regs[insn->src_reg]; -++ *dst_reg = *src_reg; -++ dst_reg->live |= REG_LIVE_WRITTEN; -++ dst_reg->subreg_def = DEF_NOT_SUBREG; -+ } else { -++ /* R1 = (u32) R2 */ -+ if (is_pointer_value(env, insn->src_reg)) { -+- verbose("R%d partial copy of pointer\n", -++ verbose(env, -++ "R%d partial copy of pointer\n", -+ insn->src_reg); -+ return -EACCES; -++ } else if (src_reg->type == SCALAR_VALUE) { -++ *dst_reg = *src_reg; -++ dst_reg->live |= REG_LIVE_WRITTEN; -++ dst_reg->subreg_def = env->insn_idx + 1; -++ } else { -++ mark_reg_unknown(env, regs, -++ insn->dst_reg); -+ } -+- regs[insn->dst_reg].type = UNKNOWN_VALUE; -+- regs[insn->dst_reg].map_ptr = NULL; -++ coerce_reg_to_size(dst_reg, 4); -+ } -+ } else { -+ /* case: R = imm -+ * remember the value we stored into this reg -+ */ -+- regs[insn->dst_reg].type = CONST_IMM; -+- regs[insn->dst_reg].imm = insn->imm; -++ /* clear any state __mark_reg_known doesn't set */ -++ mark_reg_unknown(env, regs, insn->dst_reg); -++ regs[insn->dst_reg].type = SCALAR_VALUE; -++ if (BPF_CLASS(insn->code) == BPF_ALU64) { -++ __mark_reg_known(regs + insn->dst_reg, -++ insn->imm); -++ } else { -++ __mark_reg_known(regs + insn->dst_reg, -++ (u32)insn->imm); -++ } -+ } -+ -+ } else if (opcode > BPF_END) { -+- verbose("invalid BPF_ALU opcode %x\n", opcode); -++ verbose(env, "invalid BPF_ALU opcode %x\n", opcode); -+ return -EINVAL; -+ -+ } else { /* all other ALU ops: and, sub, xor, add, ... */ -+ -+- bool stack_relative = false; -+- -+ if (BPF_SRC(insn->code) == BPF_X) { -+ if (insn->imm != 0 || insn->off != 0) { -+- verbose("BPF_ALU uses reserved fields\n"); -++ verbose(env, "BPF_ALU uses reserved fields\n"); -+ return -EINVAL; -+ } -+ /* check src1 operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ } else { -+ if (insn->src_reg != BPF_REG_0 || insn->off != 0) { -+- verbose("BPF_ALU uses reserved fields\n"); -++ verbose(env, "BPF_ALU uses reserved fields\n"); -+ return -EINVAL; -+ } -+ } -+ -+ /* check src2 operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -+ if ((opcode == BPF_MOD || opcode == BPF_DIV) && -+ BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { -+- verbose("div by zero\n"); -++ verbose(env, "div by zero\n"); -+ return -EINVAL; -+ } -+ -+@@ -1148,185 +5268,980 @@ static int check_alu_op(struct verifier_ -+ int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; -+ -+ if (insn->imm < 0 || insn->imm >= size) { -+- verbose("invalid shift %d\n", insn->imm); -++ verbose(env, "invalid shift %d\n", insn->imm); -+ return -EINVAL; -+ } -+ } -+ -+- /* pattern match 'bpf_add Rx, imm' instruction */ -+- if (opcode == BPF_ADD && BPF_CLASS(insn->code) == BPF_ALU64 && -+- regs[insn->dst_reg].type == FRAME_PTR && -+- BPF_SRC(insn->code) == BPF_K) { -+- stack_relative = true; -+- } else if (is_pointer_value(env, insn->dst_reg)) { -+- verbose("R%d pointer arithmetic prohibited\n", -+- insn->dst_reg); -+- return -EACCES; -+- } else if (BPF_SRC(insn->code) == BPF_X && -+- is_pointer_value(env, insn->src_reg)) { -+- verbose("R%d pointer arithmetic prohibited\n", -+- insn->src_reg); -+- return -EACCES; -+- } -+- -+ /* check dest operand */ -+- err = check_reg_arg(regs, insn->dst_reg, DST_OP); -++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); -+ if (err) -+ return err; -+ -+- if (stack_relative) { -+- regs[insn->dst_reg].type = PTR_TO_STACK; -+- regs[insn->dst_reg].imm = insn->imm; -+- } -++ return adjust_reg_min_max_vals(env, insn); -+ } -+ -+ return 0; -+ } -+ -+-static int check_cond_jmp_op(struct verifier_env *env, -++static void __find_good_pkt_pointers(struct bpf_func_state *state, -++ struct bpf_reg_state *dst_reg, -++ enum bpf_reg_type type, u16 new_range) -++{ -++ struct bpf_reg_state *reg; -++ int i; -++ -++ for (i = 0; i < MAX_BPF_REG; i++) { -++ reg = &state->regs[i]; -++ if (reg->type == type && reg->id == dst_reg->id) -++ /* keep the maximum range already checked */ -++ reg->range = max(reg->range, new_range); -++ } -++ -++ bpf_for_each_spilled_reg(i, state, reg) { -++ if (!reg) -++ continue; -++ if (reg->type == type && reg->id == dst_reg->id) -++ reg->range = max(reg->range, new_range); -++ } -++} -++ -++static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, -++ struct bpf_reg_state *dst_reg, -++ enum bpf_reg_type type, -++ bool range_right_open) -++{ -++ u16 new_range; -++ int i; -++ -++ if (dst_reg->off < 0 || -++ (dst_reg->off == 0 && range_right_open)) -++ /* This doesn't give us any range */ -++ return; -++ -++ if (dst_reg->umax_value > MAX_PACKET_OFF || -++ dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) -++ /* Risk of overflow. For instance, ptr + (1<<63) may be less -++ * than pkt_end, but that's because it's also less than pkt. -++ */ -++ return; -++ -++ new_range = dst_reg->off; -++ if (range_right_open) -++ new_range--; -++ -++ /* Examples for register markings: -++ * -++ * pkt_data in dst register: -++ * -++ * r2 = r3; -++ * r2 += 8; -++ * if (r2 > pkt_end) goto -++ * -++ * -++ * r2 = r3; -++ * r2 += 8; -++ * if (r2 < pkt_end) goto -++ * -++ * -++ * Where: -++ * r2 == dst_reg, pkt_end == src_reg -++ * r2=pkt(id=n,off=8,r=0) -++ * r3=pkt(id=n,off=0,r=0) -++ * -++ * pkt_data in src register: -++ * -++ * r2 = r3; -++ * r2 += 8; -++ * if (pkt_end >= r2) goto -++ * -++ * -++ * r2 = r3; -++ * r2 += 8; -++ * if (pkt_end <= r2) goto -++ * -++ * -++ * Where: -++ * pkt_end == dst_reg, r2 == src_reg -++ * r2=pkt(id=n,off=8,r=0) -++ * r3=pkt(id=n,off=0,r=0) -++ * -++ * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) -++ * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) -++ * and [r3, r3 + 8-1) respectively is safe to access depending on -++ * the check. -++ */ -++ -++ /* If our ids match, then we must have the same max_value. And we -++ * don't care about the other reg's fixed offset, since if it's too big -++ * the range won't allow anything. -++ * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. -++ */ -++ for (i = 0; i <= vstate->curframe; i++) -++ __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, -++ new_range); -++} -++ -++/* compute branch direction of the expression "if (reg opcode val) goto target;" -++ * and return: -++ * 1 - branch will be taken and "goto target" will be executed -++ * 0 - branch will not be taken and fall-through to next insn -++ * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10] -++ */ -++static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, -++ bool is_jmp32) -++{ -++ struct bpf_reg_state reg_lo; -++ s64 sval; -++ -++ if (__is_pointer_value(false, reg)) -++ return -1; -++ -++ if (is_jmp32) { -++ reg_lo = *reg; -++ reg = ®_lo; -++ /* For JMP32, only low 32 bits are compared, coerce_reg_to_size -++ * could truncate high bits and update umin/umax according to -++ * information of low bits. -++ */ -++ coerce_reg_to_size(reg, 4); -++ /* smin/smax need special handling. For example, after coerce, -++ * if smin_value is 0x00000000ffffffffLL, the value is -1 when -++ * used as operand to JMP32. It is a negative number from s32's -++ * point of view, while it is a positive number when seen as -++ * s64. The smin/smax are kept as s64, therefore, when used with -++ * JMP32, they need to be transformed into s32, then sign -++ * extended back to s64. -++ * -++ * Also, smin/smax were copied from umin/umax. If umin/umax has -++ * different sign bit, then min/max relationship doesn't -++ * maintain after casting into s32, for this case, set smin/smax -++ * to safest range. -++ */ -++ if ((reg->umax_value ^ reg->umin_value) & -++ (1ULL << 31)) { -++ reg->smin_value = S32_MIN; -++ reg->smax_value = S32_MAX; -++ } -++ reg->smin_value = (s64)(s32)reg->smin_value; -++ reg->smax_value = (s64)(s32)reg->smax_value; -++ -++ val = (u32)val; -++ sval = (s64)(s32)val; -++ } else { -++ sval = (s64)val; -++ } -++ -++ switch (opcode) { -++ case BPF_JEQ: -++ if (tnum_is_const(reg->var_off)) -++ return !!tnum_equals_const(reg->var_off, val); -++ break; -++ case BPF_JNE: -++ if (tnum_is_const(reg->var_off)) -++ return !tnum_equals_const(reg->var_off, val); -++ break; -++ case BPF_JSET: -++ if ((~reg->var_off.mask & reg->var_off.value) & val) -++ return 1; -++ if (!((reg->var_off.mask | reg->var_off.value) & val)) -++ return 0; -++ break; -++ case BPF_JGT: -++ if (reg->umin_value > val) -++ return 1; -++ else if (reg->umax_value <= val) -++ return 0; -++ break; -++ case BPF_JSGT: -++ if (reg->smin_value > sval) -++ return 1; -++ else if (reg->smax_value < sval) -++ return 0; -++ break; -++ case BPF_JLT: -++ if (reg->umax_value < val) -++ return 1; -++ else if (reg->umin_value >= val) -++ return 0; -++ break; -++ case BPF_JSLT: -++ if (reg->smax_value < sval) -++ return 1; -++ else if (reg->smin_value >= sval) -++ return 0; -++ break; -++ case BPF_JGE: -++ if (reg->umin_value >= val) -++ return 1; -++ else if (reg->umax_value < val) -++ return 0; -++ break; -++ case BPF_JSGE: -++ if (reg->smin_value >= sval) -++ return 1; -++ else if (reg->smax_value < sval) -++ return 0; -++ break; -++ case BPF_JLE: -++ if (reg->umax_value <= val) -++ return 1; -++ else if (reg->umin_value > val) -++ return 0; -++ break; -++ case BPF_JSLE: -++ if (reg->smax_value <= sval) -++ return 1; -++ else if (reg->smin_value > sval) -++ return 0; -++ break; -++ } -++ -++ return -1; -++} -++ -++/* Generate min value of the high 32-bit from TNUM info. */ -++static u64 gen_hi_min(struct tnum var) -++{ -++ return var.value & ~0xffffffffULL; -++} -++ -++/* Generate max value of the high 32-bit from TNUM info. */ -++static u64 gen_hi_max(struct tnum var) -++{ -++ return (var.value | var.mask) & ~0xffffffffULL; -++} -++ -++/* Return true if VAL is compared with a s64 sign extended from s32, and they -++ * are with the same signedness. -++ */ -++static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg) -++{ -++ return ((s32)sval >= 0 && -++ reg->smin_value >= 0 && reg->smax_value <= S32_MAX) || -++ ((s32)sval < 0 && -++ reg->smax_value <= 0 && reg->smin_value >= S32_MIN); -++} -++ -++/* Constrain the possible values of @reg with unsigned upper bound @bound. -++ * If @is_exclusive, @bound is an exclusive limit, otherwise it is inclusive. -++ * If @is_jmp32, @bound is a 32-bit value that only constrains the low 32 bits -++ * of @reg. -++ */ -++static void set_upper_bound(struct bpf_reg_state *reg, u64 bound, bool is_jmp32, -++ bool is_exclusive) -++{ -++ if (is_exclusive) { -++ /* There are no values for `reg` that make `reg<0` true. */ -++ if (bound == 0) -++ return; -++ bound--; -++ } -++ if (is_jmp32) { -++ /* Constrain the register's value in the tnum representation. -++ * For 64-bit comparisons this happens later in -++ * __reg_bound_offset(), but for 32-bit comparisons, we can be -++ * more precise than what can be derived from the updated -++ * numeric bounds. -++ */ -++ struct tnum t = tnum_range(0, bound); -++ -++ t.mask |= ~0xffffffffULL; /* upper half is unknown */ -++ reg->var_off = tnum_intersect(reg->var_off, t); -++ -++ /* Compute the 64-bit bound from the 32-bit bound. */ -++ bound += gen_hi_max(reg->var_off); -++ } -++ reg->umax_value = min(reg->umax_value, bound); -++} -++ -++/* Constrain the possible values of @reg with unsigned lower bound @bound. -++ * If @is_exclusive, @bound is an exclusive limit, otherwise it is inclusive. -++ * If @is_jmp32, @bound is a 32-bit value that only constrains the low 32 bits -++ * of @reg. -++ */ -++static void set_lower_bound(struct bpf_reg_state *reg, u64 bound, bool is_jmp32, -++ bool is_exclusive) -++{ -++ if (is_exclusive) { -++ /* There are no values for `reg` that make `reg>MAX` true. */ -++ if (bound == (is_jmp32 ? U32_MAX : U64_MAX)) -++ return; -++ bound++; -++ } -++ if (is_jmp32) { -++ /* Constrain the register's value in the tnum representation. -++ * For 64-bit comparisons this happens later in -++ * __reg_bound_offset(), but for 32-bit comparisons, we can be -++ * more precise than what can be derived from the updated -++ * numeric bounds. -++ */ -++ struct tnum t = tnum_range(bound, U32_MAX); -++ -++ t.mask |= ~0xffffffffULL; /* upper half is unknown */ -++ reg->var_off = tnum_intersect(reg->var_off, t); -++ -++ /* Compute the 64-bit bound from the 32-bit bound. */ -++ bound += gen_hi_min(reg->var_off); -++ } -++ reg->umin_value = max(reg->umin_value, bound); -++} -++ -++/* Adjusts the register min/max values in the case that the dst_reg is the -++ * variable register that we are working on, and src_reg is a constant or we're -++ * simply doing a BPF_K check. -++ * In JEQ/JNE cases we also adjust the var_off values. -++ */ -++static void reg_set_min_max(struct bpf_reg_state *true_reg, -++ struct bpf_reg_state *false_reg, u64 val, -++ u8 opcode, bool is_jmp32) -++{ -++ s64 sval; -++ -++ /* If the dst_reg is a pointer, we can't learn anything about its -++ * variable offset from the compare (unless src_reg were a pointer into -++ * the same object, but we don't bother with that. -++ * Since false_reg and true_reg have the same type by construction, we -++ * only need to check one of them for pointerness. -++ */ -++ if (__is_pointer_value(false, false_reg)) -++ return; -++ -++ val = is_jmp32 ? (u32)val : val; -++ sval = is_jmp32 ? (s64)(s32)val : (s64)val; -++ -++ switch (opcode) { -++ case BPF_JEQ: -++ case BPF_JNE: -++ { -++ struct bpf_reg_state *reg = -++ opcode == BPF_JEQ ? true_reg : false_reg; -++ -++ /* For BPF_JEQ, if this is false we know nothing Jon Snow, but -++ * if it is true we know the value for sure. Likewise for -++ * BPF_JNE. -++ */ -++ if (is_jmp32) { -++ u64 old_v = reg->var_off.value; -++ u64 hi_mask = ~0xffffffffULL; -++ -++ reg->var_off.value = (old_v & hi_mask) | val; -++ reg->var_off.mask &= hi_mask; -++ } else { -++ __mark_reg_known(reg, val); -++ } -++ break; -++ } -++ case BPF_JSET: -++ false_reg->var_off = tnum_and(false_reg->var_off, -++ tnum_const(~val)); -++ if (is_power_of_2(val)) -++ true_reg->var_off = tnum_or(true_reg->var_off, -++ tnum_const(val)); -++ break; -++ case BPF_JGE: -++ case BPF_JGT: -++ { -++ set_upper_bound(false_reg, val, is_jmp32, opcode == BPF_JGE); -++ set_lower_bound(true_reg, val, is_jmp32, opcode == BPF_JGT); -++ break; -++ } -++ case BPF_JSGE: -++ case BPF_JSGT: -++ { -++ s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; -++ s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; -++ -++ /* If the full s64 was not sign-extended from s32 then don't -++ * deduct further info. -++ */ -++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) -++ break; -++ false_reg->smax_value = min(false_reg->smax_value, false_smax); -++ true_reg->smin_value = max(true_reg->smin_value, true_smin); -++ break; -++ } -++ case BPF_JLE: -++ case BPF_JLT: -++ { -++ set_lower_bound(false_reg, val, is_jmp32, opcode == BPF_JLE); -++ set_upper_bound(true_reg, val, is_jmp32, opcode == BPF_JLT); -++ break; -++ } -++ case BPF_JSLE: -++ case BPF_JSLT: -++ { -++ s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; -++ s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; -++ -++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) -++ break; -++ false_reg->smin_value = max(false_reg->smin_value, false_smin); -++ true_reg->smax_value = min(true_reg->smax_value, true_smax); -++ break; -++ } -++ default: -++ break; -++ } -++ -++ __reg_deduce_bounds(false_reg); -++ __reg_deduce_bounds(true_reg); -++ /* We might have learned some bits from the bounds. */ -++ __reg_bound_offset(false_reg); -++ __reg_bound_offset(true_reg); -++ /* Intersecting with the old var_off might have improved our bounds -++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), -++ * then new var_off is (0; 0x7f...fc) which improves our umax. -++ */ -++ __update_reg_bounds(false_reg); -++ __update_reg_bounds(true_reg); -++} -++ -++/* Same as above, but for the case that dst_reg holds a constant and src_reg is -++ * the variable reg. -++ */ -++static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, -++ struct bpf_reg_state *false_reg, u64 val, -++ u8 opcode, bool is_jmp32) -++{ -++ s64 sval; -++ -++ if (__is_pointer_value(false, false_reg)) -++ return; -++ -++ val = is_jmp32 ? (u32)val : val; -++ sval = is_jmp32 ? (s64)(s32)val : (s64)val; -++ -++ switch (opcode) { -++ case BPF_JEQ: -++ case BPF_JNE: -++ { -++ struct bpf_reg_state *reg = -++ opcode == BPF_JEQ ? true_reg : false_reg; -++ -++ if (is_jmp32) { -++ u64 old_v = reg->var_off.value; -++ u64 hi_mask = ~0xffffffffULL; -++ -++ reg->var_off.value = (old_v & hi_mask) | val; -++ reg->var_off.mask &= hi_mask; -++ } else { -++ __mark_reg_known(reg, val); -++ } -++ break; -++ } -++ case BPF_JSET: -++ false_reg->var_off = tnum_and(false_reg->var_off, -++ tnum_const(~val)); -++ if (is_power_of_2(val)) -++ true_reg->var_off = tnum_or(true_reg->var_off, -++ tnum_const(val)); -++ break; -++ case BPF_JGE: -++ case BPF_JGT: -++ { -++ set_lower_bound(false_reg, val, is_jmp32, opcode == BPF_JGE); -++ set_upper_bound(true_reg, val, is_jmp32, opcode == BPF_JGT); -++ break; -++ } -++ case BPF_JSGE: -++ case BPF_JSGT: -++ { -++ s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1; -++ s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval; -++ -++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) -++ break; -++ false_reg->smin_value = max(false_reg->smin_value, false_smin); -++ true_reg->smax_value = min(true_reg->smax_value, true_smax); -++ break; -++ } -++ case BPF_JLE: -++ case BPF_JLT: -++ { -++ set_upper_bound(false_reg, val, is_jmp32, opcode == BPF_JLE); -++ set_lower_bound(true_reg, val, is_jmp32, opcode == BPF_JLT); -++ break; -++ } -++ case BPF_JSLE: -++ case BPF_JSLT: -++ { -++ s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1; -++ s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval; -++ -++ if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) -++ break; -++ false_reg->smax_value = min(false_reg->smax_value, false_smax); -++ true_reg->smin_value = max(true_reg->smin_value, true_smin); -++ break; -++ } -++ default: -++ break; -++ } -++ -++ __reg_deduce_bounds(false_reg); -++ __reg_deduce_bounds(true_reg); -++ /* We might have learned some bits from the bounds. */ -++ __reg_bound_offset(false_reg); -++ __reg_bound_offset(true_reg); -++ /* Intersecting with the old var_off might have improved our bounds -++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), -++ * then new var_off is (0; 0x7f...fc) which improves our umax. -++ */ -++ __update_reg_bounds(false_reg); -++ __update_reg_bounds(true_reg); -++} -++ -++/* Regs are known to be equal, so intersect their min/max/var_off */ -++static void __reg_combine_min_max(struct bpf_reg_state *src_reg, -++ struct bpf_reg_state *dst_reg) -++{ -++ src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, -++ dst_reg->umin_value); -++ src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, -++ dst_reg->umax_value); -++ src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, -++ dst_reg->smin_value); -++ src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, -++ dst_reg->smax_value); -++ src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, -++ dst_reg->var_off); -++ /* We might have learned new bounds from the var_off. */ -++ __update_reg_bounds(src_reg); -++ __update_reg_bounds(dst_reg); -++ /* We might have learned something about the sign bit. */ -++ __reg_deduce_bounds(src_reg); -++ __reg_deduce_bounds(dst_reg); -++ /* We might have learned some bits from the bounds. */ -++ __reg_bound_offset(src_reg); -++ __reg_bound_offset(dst_reg); -++ /* Intersecting with the old var_off might have improved our bounds -++ * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), -++ * then new var_off is (0; 0x7f...fc) which improves our umax. -++ */ -++ __update_reg_bounds(src_reg); -++ __update_reg_bounds(dst_reg); -++} -++ -++static void reg_combine_min_max(struct bpf_reg_state *true_src, -++ struct bpf_reg_state *true_dst, -++ struct bpf_reg_state *false_src, -++ struct bpf_reg_state *false_dst, -++ u8 opcode) -++{ -++ switch (opcode) { -++ case BPF_JEQ: -++ __reg_combine_min_max(true_src, true_dst); -++ break; -++ case BPF_JNE: -++ __reg_combine_min_max(false_src, false_dst); -++ break; -++ } -++} -++ -++static void mark_ptr_or_null_reg(struct bpf_func_state *state, -++ struct bpf_reg_state *reg, u32 id, -++ bool is_null) -++{ -++ if (reg_type_may_be_null(reg->type) && reg->id == id) { -++ /* Old offset (both fixed and variable parts) should -++ * have been known-zero, because we don't allow pointer -++ * arithmetic on pointers that might be NULL. -++ */ -++ if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || -++ !tnum_equals_const(reg->var_off, 0) || -++ reg->off)) { -++ __mark_reg_known_zero(reg); -++ reg->off = 0; -++ } -++ if (is_null) { -++ reg->type = SCALAR_VALUE; -++ } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) { -++ if (reg->map_ptr->inner_map_meta) { -++ reg->type = CONST_PTR_TO_MAP; -++ reg->map_ptr = reg->map_ptr->inner_map_meta; -++ } else if (reg->map_ptr->map_type == -++ BPF_MAP_TYPE_XSKMAP) { -++ reg->type = PTR_TO_XDP_SOCK; -++ } else { -++ reg->type = PTR_TO_MAP_VALUE; -++ } -++ } else if (reg->type == PTR_TO_SOCKET_OR_NULL) { -++ reg->type = PTR_TO_SOCKET; -++ } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) { -++ reg->type = PTR_TO_SOCK_COMMON; -++ } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { -++ reg->type = PTR_TO_TCP_SOCK; -++ } -++ if (is_null) { -++ /* We don't need id and ref_obj_id from this point -++ * onwards anymore, thus we should better reset it, -++ * so that state pruning has chances to take effect. -++ */ -++ reg->id = 0; -++ reg->ref_obj_id = 0; -++ } else if (!reg_may_point_to_spin_lock(reg)) { -++ /* For not-NULL ptr, reg->ref_obj_id will be reset -++ * in release_reg_references(). -++ * -++ * reg->id is still used by spin_lock ptr. Other -++ * than spin_lock ptr type, reg->id can be reset. -++ */ -++ reg->id = 0; -++ } -++ } -++} -++ -++static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, -++ bool is_null) -++{ -++ struct bpf_reg_state *reg; -++ int i; -++ -++ for (i = 0; i < MAX_BPF_REG; i++) -++ mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); -++ -++ bpf_for_each_spilled_reg(i, state, reg) { -++ if (!reg) -++ continue; -++ mark_ptr_or_null_reg(state, reg, id, is_null); -++ } -++} -++ -++/* The logic is similar to find_good_pkt_pointers(), both could eventually -++ * be folded together at some point. -++ */ -++static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, -++ bool is_null) -++{ -++ struct bpf_func_state *state = vstate->frame[vstate->curframe]; -++ struct bpf_reg_state *regs = state->regs; -++ u32 ref_obj_id = regs[regno].ref_obj_id; -++ u32 id = regs[regno].id; -++ int i; -++ -++ if (ref_obj_id && ref_obj_id == id && is_null) -++ /* regs[regno] is in the " == NULL" branch. -++ * No one could have freed the reference state before -++ * doing the NULL check. -++ */ -++ WARN_ON_ONCE(release_reference_state(state, id)); -++ -++ for (i = 0; i <= vstate->curframe; i++) -++ __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); -++} -++ -++static bool try_match_pkt_pointers(const struct bpf_insn *insn, -++ struct bpf_reg_state *dst_reg, -++ struct bpf_reg_state *src_reg, -++ struct bpf_verifier_state *this_branch, -++ struct bpf_verifier_state *other_branch) -++{ -++ if (BPF_SRC(insn->code) != BPF_X) -++ return false; -++ -++ /* Pointers are always 64-bit. */ -++ if (BPF_CLASS(insn->code) == BPF_JMP32) -++ return false; -++ -++ switch (BPF_OP(insn->code)) { -++ case BPF_JGT: -++ if ((dst_reg->type == PTR_TO_PACKET && -++ src_reg->type == PTR_TO_PACKET_END) || -++ (dst_reg->type == PTR_TO_PACKET_META && -++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { -++ /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ -++ find_good_pkt_pointers(this_branch, dst_reg, -++ dst_reg->type, false); -++ } else if ((dst_reg->type == PTR_TO_PACKET_END && -++ src_reg->type == PTR_TO_PACKET) || -++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && -++ src_reg->type == PTR_TO_PACKET_META)) { -++ /* pkt_end > pkt_data', pkt_data > pkt_meta' */ -++ find_good_pkt_pointers(other_branch, src_reg, -++ src_reg->type, true); -++ } else { -++ return false; -++ } -++ break; -++ case BPF_JLT: -++ if ((dst_reg->type == PTR_TO_PACKET && -++ src_reg->type == PTR_TO_PACKET_END) || -++ (dst_reg->type == PTR_TO_PACKET_META && -++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { -++ /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ -++ find_good_pkt_pointers(other_branch, dst_reg, -++ dst_reg->type, true); -++ } else if ((dst_reg->type == PTR_TO_PACKET_END && -++ src_reg->type == PTR_TO_PACKET) || -++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && -++ src_reg->type == PTR_TO_PACKET_META)) { -++ /* pkt_end < pkt_data', pkt_data > pkt_meta' */ -++ find_good_pkt_pointers(this_branch, src_reg, -++ src_reg->type, false); -++ } else { -++ return false; -++ } -++ break; -++ case BPF_JGE: -++ if ((dst_reg->type == PTR_TO_PACKET && -++ src_reg->type == PTR_TO_PACKET_END) || -++ (dst_reg->type == PTR_TO_PACKET_META && -++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { -++ /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ -++ find_good_pkt_pointers(this_branch, dst_reg, -++ dst_reg->type, true); -++ } else if ((dst_reg->type == PTR_TO_PACKET_END && -++ src_reg->type == PTR_TO_PACKET) || -++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && -++ src_reg->type == PTR_TO_PACKET_META)) { -++ /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ -++ find_good_pkt_pointers(other_branch, src_reg, -++ src_reg->type, false); -++ } else { -++ return false; -++ } -++ break; -++ case BPF_JLE: -++ if ((dst_reg->type == PTR_TO_PACKET && -++ src_reg->type == PTR_TO_PACKET_END) || -++ (dst_reg->type == PTR_TO_PACKET_META && -++ reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { -++ /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ -++ find_good_pkt_pointers(other_branch, dst_reg, -++ dst_reg->type, false); -++ } else if ((dst_reg->type == PTR_TO_PACKET_END && -++ src_reg->type == PTR_TO_PACKET) || -++ (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && -++ src_reg->type == PTR_TO_PACKET_META)) { -++ /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ -++ find_good_pkt_pointers(this_branch, src_reg, -++ src_reg->type, true); -++ } else { -++ return false; -++ } -++ break; -++ default: -++ return false; -++ } -++ -++ return true; -++} -++ -++static int check_cond_jmp_op(struct bpf_verifier_env *env, -+ struct bpf_insn *insn, int *insn_idx) -+ { -+- struct reg_state *regs = env->cur_state.regs; -+- struct verifier_state *other_branch; -++ struct bpf_verifier_state *this_branch = env->cur_state; -++ struct bpf_verifier_state *other_branch; -++ struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; -++ struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; -+ u8 opcode = BPF_OP(insn->code); -++ bool is_jmp32; -++ int pred = -1; -+ int err; -+ -+- if (opcode > BPF_EXIT) { -+- verbose("invalid BPF_JMP opcode %x\n", opcode); -++ /* Only conditional jumps are expected to reach here. */ -++ if (opcode == BPF_JA || opcode > BPF_JSLE) { -++ verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); -+ return -EINVAL; -+ } -+ -+ if (BPF_SRC(insn->code) == BPF_X) { -+ if (insn->imm != 0) { -+- verbose("BPF_JMP uses reserved fields\n"); -++ verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ /* check src1 operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ -+ if (is_pointer_value(env, insn->src_reg)) { -+- verbose("R%d pointer comparison prohibited\n", -++ verbose(env, "R%d pointer comparison prohibited\n", -+ insn->src_reg); -+ return -EACCES; -+ } -++ src_reg = ®s[insn->src_reg]; -+ } else { -+ if (insn->src_reg != BPF_REG_0) { -+- verbose("BPF_JMP uses reserved fields\n"); -++ verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); -+ return -EINVAL; -+ } -+ } -+ -+ /* check src2 operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -+- /* detect if R == 0 where R was initialized to zero earlier */ -+- if (BPF_SRC(insn->code) == BPF_K && -+- (opcode == BPF_JEQ || opcode == BPF_JNE) && -+- regs[insn->dst_reg].type == CONST_IMM && -+- regs[insn->dst_reg].imm == insn->imm) { -+- if (opcode == BPF_JEQ) { -+- /* if (imm == imm) goto pc+off; -+- * only follow the goto, ignore fall-through -+- */ -+- *insn_idx += insn->off; -+- return 0; -+- } else { -+- /* if (imm != imm) goto pc+off; -+- * only follow fall-through branch, since -+- * that's where the program will go -+- */ -+- return 0; -+- } -++ dst_reg = ®s[insn->dst_reg]; -++ is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; -++ -++ if (BPF_SRC(insn->code) == BPF_K) -++ pred = is_branch_taken(dst_reg, insn->imm, -++ opcode, is_jmp32); -++ else if (src_reg->type == SCALAR_VALUE && -++ tnum_is_const(src_reg->var_off)) -++ pred = is_branch_taken(dst_reg, src_reg->var_off.value, -++ opcode, is_jmp32); -++ if (pred >= 0) { -++ err = mark_chain_precision(env, insn->dst_reg); -++ if (BPF_SRC(insn->code) == BPF_X && !err) -++ err = mark_chain_precision(env, insn->src_reg); -++ if (err) -++ return err; -++ } -++ -++ if (pred == 1) { -++ /* Only follow the goto, ignore fall-through. If needed, push -++ * the fall-through branch for simulation under speculative -++ * execution. -++ */ -++ if (!env->allow_ptr_leaks && -++ !sanitize_speculative_path(env, insn, *insn_idx + 1, -++ *insn_idx)) -++ return -EFAULT; -++ *insn_idx += insn->off; -++ return 0; -++ } else if (pred == 0) { -++ /* Only follow the fall-through branch, since that's where the -++ * program will go. If needed, push the goto branch for -++ * simulation under speculative execution. -++ */ -++ if (!env->allow_ptr_leaks && -++ !sanitize_speculative_path(env, insn, -++ *insn_idx + insn->off + 1, -++ *insn_idx)) -++ return -EFAULT; -++ return 0; -+ } -+ -+- other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx); -++ other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, -++ false); -+ if (!other_branch) -+ return -EFAULT; -++ other_branch_regs = other_branch->frame[other_branch->curframe]->regs; -+ -+- /* detect if R == 0 where R is returned value from bpf_map_lookup_elem() */ -+- if (BPF_SRC(insn->code) == BPF_K && -+- insn->imm == 0 && (opcode == BPF_JEQ || -+- opcode == BPF_JNE) && -+- regs[insn->dst_reg].type == PTR_TO_MAP_VALUE_OR_NULL) { -+- if (opcode == BPF_JEQ) { -+- /* next fallthrough insn can access memory via -+- * this register -+- */ -+- regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; -+- /* branch targer cannot access it, since reg == 0 */ -+- other_branch->regs[insn->dst_reg].type = CONST_IMM; -+- other_branch->regs[insn->dst_reg].imm = 0; -+- } else { -+- other_branch->regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; -+- regs[insn->dst_reg].type = CONST_IMM; -+- regs[insn->dst_reg].imm = 0; -+- } -+- } else if (is_pointer_value(env, insn->dst_reg)) { -+- verbose("R%d pointer comparison prohibited\n", insn->dst_reg); -++ /* detect if we are comparing against a constant value so we can adjust -++ * our min/max values for our dst register. -++ * this is only legit if both are scalars (or pointers to the same -++ * object, I suppose, but we don't support that right now), because -++ * otherwise the different base pointers mean the offsets aren't -++ * comparable. -++ */ -++ if (BPF_SRC(insn->code) == BPF_X) { -++ struct bpf_reg_state *src_reg = ®s[insn->src_reg]; -++ struct bpf_reg_state lo_reg0 = *dst_reg; -++ struct bpf_reg_state lo_reg1 = *src_reg; -++ struct bpf_reg_state *src_lo, *dst_lo; -++ -++ dst_lo = &lo_reg0; -++ src_lo = &lo_reg1; -++ coerce_reg_to_size(dst_lo, 4); -++ coerce_reg_to_size(src_lo, 4); -++ -++ if (dst_reg->type == SCALAR_VALUE && -++ src_reg->type == SCALAR_VALUE) { -++ if (tnum_is_const(src_reg->var_off) || -++ (is_jmp32 && tnum_is_const(src_lo->var_off))) -++ reg_set_min_max(&other_branch_regs[insn->dst_reg], -++ dst_reg, -++ is_jmp32 -++ ? src_lo->var_off.value -++ : src_reg->var_off.value, -++ opcode, is_jmp32); -++ else if (tnum_is_const(dst_reg->var_off) || -++ (is_jmp32 && tnum_is_const(dst_lo->var_off))) -++ reg_set_min_max_inv(&other_branch_regs[insn->src_reg], -++ src_reg, -++ is_jmp32 -++ ? dst_lo->var_off.value -++ : dst_reg->var_off.value, -++ opcode, is_jmp32); -++ else if (!is_jmp32 && -++ (opcode == BPF_JEQ || opcode == BPF_JNE)) -++ /* Comparing for equality, we can combine knowledge */ -++ reg_combine_min_max(&other_branch_regs[insn->src_reg], -++ &other_branch_regs[insn->dst_reg], -++ src_reg, dst_reg, opcode); -++ } -++ } else if (dst_reg->type == SCALAR_VALUE) { -++ reg_set_min_max(&other_branch_regs[insn->dst_reg], -++ dst_reg, insn->imm, opcode, is_jmp32); -++ } -++ -++ /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). -++ * NOTE: these optimizations below are related with pointer comparison -++ * which will never be JMP32. -++ */ -++ if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && -++ insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && -++ reg_type_may_be_null(dst_reg->type)) { -++ /* Mark all identical registers in each branch as either -++ * safe or unknown depending R == 0 or R != 0 conditional. -++ */ -++ mark_ptr_or_null_regs(this_branch, insn->dst_reg, -++ opcode == BPF_JNE); -++ mark_ptr_or_null_regs(other_branch, insn->dst_reg, -++ opcode == BPF_JEQ); -++ } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], -++ this_branch, other_branch) && -++ is_pointer_value(env, insn->dst_reg)) { -++ verbose(env, "R%d pointer comparison prohibited\n", -++ insn->dst_reg); -+ return -EACCES; -+- } else if (BPF_SRC(insn->code) == BPF_K && -+- (opcode == BPF_JEQ || opcode == BPF_JNE)) { -+- -+- if (opcode == BPF_JEQ) { -+- /* detect if (R == imm) goto -+- * and in the target state recognize that R = imm -+- */ -+- other_branch->regs[insn->dst_reg].type = CONST_IMM; -+- other_branch->regs[insn->dst_reg].imm = insn->imm; -+- } else { -+- /* detect if (R != imm) goto -+- * and in the fall-through state recognize that R = imm -+- */ -+- regs[insn->dst_reg].type = CONST_IMM; -+- regs[insn->dst_reg].imm = insn->imm; -+- } -+ } -+- if (log_level) -+- print_verifier_state(env); -++ if (env->log.level & BPF_LOG_LEVEL) -++ print_verifier_state(env, this_branch->frame[this_branch->curframe]); -+ return 0; -+ } -+ -+-/* return the map pointer stored inside BPF_LD_IMM64 instruction */ -+-static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn) -+-{ -+- u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32; -+- -+- return (struct bpf_map *) (unsigned long) imm64; -+-} -+- -+ /* verify BPF_LD_IMM64 instruction */ -+-static int check_ld_imm(struct verifier_env *env, struct bpf_insn *insn) -++static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) -+ { -+- struct reg_state *regs = env->cur_state.regs; -++ struct bpf_insn_aux_data *aux = cur_aux(env); -++ struct bpf_reg_state *regs = cur_regs(env); -++ struct bpf_map *map; -+ int err; -+ -+ if (BPF_SIZE(insn->code) != BPF_DW) { -+- verbose("invalid BPF_LD_IMM insn\n"); -++ verbose(env, "invalid BPF_LD_IMM insn\n"); -+ return -EINVAL; -+ } -+ if (insn->off != 0) { -+- verbose("BPF_LD_IMM64 uses reserved fields\n"); -++ verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+- err = check_reg_arg(regs, insn->dst_reg, DST_OP); -++ err = check_reg_arg(env, insn->dst_reg, DST_OP); -+ if (err) -+ return err; -+ -+- if (insn->src_reg == 0) -+- /* generic move 64-bit immediate into a register */ -++ if (insn->src_reg == 0) { -++ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; -++ -++ regs[insn->dst_reg].type = SCALAR_VALUE; -++ __mark_reg_known(®s[insn->dst_reg], imm); -+ return 0; -++ } -+ -+- /* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */ -+- BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD); -++ map = env->used_maps[aux->map_index]; -++ mark_reg_known_zero(env, regs, insn->dst_reg); -++ regs[insn->dst_reg].map_ptr = map; -++ -++ if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { -++ regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; -++ regs[insn->dst_reg].off = aux->map_off; -++ if (map_value_has_spin_lock(map)) -++ regs[insn->dst_reg].id = ++env->id_gen; -++ } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { -++ regs[insn->dst_reg].type = CONST_PTR_TO_MAP; -++ } else { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } -+ -+- regs[insn->dst_reg].type = CONST_PTR_TO_MAP; -+- regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn); -+ return 0; -+ } -+ -+@@ -1357,53 +6272,148 @@ static bool may_access_skb(enum bpf_prog -+ * Output: -+ * R0 - 8/16/32-bit skb data converted to cpu endianness -+ */ -+-static int check_ld_abs(struct verifier_env *env, struct bpf_insn *insn) -++static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) -+ { -+- struct reg_state *regs = env->cur_state.regs; -++ struct bpf_reg_state *regs = cur_regs(env); -++ static const int ctx_reg = BPF_REG_6; -+ u8 mode = BPF_MODE(insn->code); -+- struct reg_state *reg; -+ int i, err; -+ -+ if (!may_access_skb(env->prog->type)) { -+- verbose("BPF_LD_ABS|IND instructions not allowed for this program type\n"); -++ verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); -++ return -EINVAL; -++ } -++ -++ if (!env->ops->gen_ld_abs) { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } -++ -++ if (env->subprog_cnt > 1) { -++ /* when program has LD_ABS insn JITs and interpreter assume -++ * that r1 == ctx == skb which is not the case for callees -++ * that can have arbitrary arguments. It's problematic -++ * for main prog as well since JITs would need to analyze -++ * all functions in order to make proper register save/restore -++ * decisions in the main prog. Hence disallow LD_ABS with calls -++ */ -++ verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); -+ return -EINVAL; -+ } -+ -+ if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || -+ BPF_SIZE(insn->code) == BPF_DW || -+ (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { -+- verbose("BPF_LD_ABS uses reserved fields\n"); -++ verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ /* check whether implicit source operand (register R6) is readable */ -+- err = check_reg_arg(regs, BPF_REG_6, SRC_OP); -++ err = check_reg_arg(env, ctx_reg, SRC_OP); -+ if (err) -+ return err; -+ -+- if (regs[BPF_REG_6].type != PTR_TO_CTX) { -+- verbose("at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); -++ /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as -++ * gen_ld_abs() may terminate the program at runtime, leading to -++ * reference leak. -++ */ -++ err = check_reference_leak(env); -++ if (err) { -++ verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); -++ return err; -++ } -++ -++ if (env->cur_state->active_spin_lock) { -++ verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); -++ return -EINVAL; -++ } -++ -++ if (regs[ctx_reg].type != PTR_TO_CTX) { -++ verbose(env, -++ "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); -+ return -EINVAL; -+ } -+ -+ if (mode == BPF_IND) { -+ /* check explicit source operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ } -+ -++ err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); -++ if (err < 0) -++ return err; -++ -+ /* reset caller saved regs to unreadable */ -+ for (i = 0; i < CALLER_SAVED_REGS; i++) { -+- reg = regs + caller_saved[i]; -+- reg->type = NOT_INIT; -+- reg->imm = 0; -++ mark_reg_not_init(env, regs, caller_saved[i]); -++ check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); -+ } -+ -+ /* mark destination R0 register as readable, since it contains -+- * the value fetched from the packet -++ * the value fetched from the packet. -++ * Already marked as written above. -+ */ -+- regs[BPF_REG_0].type = UNKNOWN_VALUE; -++ mark_reg_unknown(env, regs, BPF_REG_0); -++ /* ld_abs load up to 32-bit skb data. */ -++ regs[BPF_REG_0].subreg_def = env->insn_idx + 1; -++ return 0; -++} -++ -++static int check_return_code(struct bpf_verifier_env *env) -++{ -++ struct tnum enforce_attach_type_range = tnum_unknown; -++ struct bpf_reg_state *reg; -++ struct tnum range = tnum_range(0, 1); -++ -++ switch (env->prog->type) { -++ case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: -++ if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || -++ env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG) -++ range = tnum_range(1, 1); -++ break; -++ case BPF_PROG_TYPE_CGROUP_SKB: -++ if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { -++ range = tnum_range(0, 3); -++ enforce_attach_type_range = tnum_range(2, 3); -++ } -++ break; -++ case BPF_PROG_TYPE_CGROUP_SOCK: -++ case BPF_PROG_TYPE_SOCK_OPS: -++ case BPF_PROG_TYPE_CGROUP_DEVICE: -++ case BPF_PROG_TYPE_CGROUP_SYSCTL: -++ case BPF_PROG_TYPE_CGROUP_SOCKOPT: -++ break; -++ default: -++ return 0; -++ } -++ -++ reg = cur_regs(env) + BPF_REG_0; -++ if (reg->type != SCALAR_VALUE) { -++ verbose(env, "At program exit the register R0 is not a known value (%s)\n", -++ reg_type_str[reg->type]); -++ return -EINVAL; -++ } -++ -++ if (!tnum_in(range, reg->var_off)) { -++ char tn_buf[48]; -++ -++ verbose(env, "At program exit the register R0 "); -++ if (!tnum_is_unknown(reg->var_off)) { -++ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); -++ verbose(env, "has value %s", tn_buf); -++ } else { -++ verbose(env, "has unknown scalar value"); -++ } -++ tnum_strn(tn_buf, sizeof(tn_buf), range); -++ verbose(env, " should have been in %s\n", tn_buf); -++ return -EINVAL; -++ } -++ -++ if (!tnum_is_unknown(enforce_attach_type_range) && -++ tnum_in(enforce_attach_type_range, reg->var_off)) -++ env->prog->enforce_expected_attach_type = 1; -+ return 0; -+ } -+ -+@@ -1447,19 +6457,37 @@ enum { -+ BRANCH = 2, -+ }; -+ -+-#define STATE_LIST_MARK ((struct verifier_state_list *) -1L) -++static u32 state_htab_size(struct bpf_verifier_env *env) -++{ -++ return env->prog->len; -++} -++ -++static struct bpf_verifier_state_list **explored_state( -++ struct bpf_verifier_env *env, -++ int idx) -++{ -++ struct bpf_verifier_state *cur = env->cur_state; -++ struct bpf_func_state *state = cur->frame[cur->curframe]; -++ -++ return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; -++} -+ -+-static int *insn_stack; /* stack of insns to process */ -+-static int cur_stack; /* current stack index */ -+-static int *insn_state; -++static void init_explored_state(struct bpf_verifier_env *env, int idx) -++{ -++ env->insn_aux_data[idx].prune_point = true; -++} -+ -+ /* t, w, e - match pseudo-code above: -+ * t - index of current instruction -+ * w - next instruction -+ * e - edge -+ */ -+-static int push_insn(int t, int w, int e, struct verifier_env *env) -++static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, -++ bool loop_ok) -+ { -++ int *insn_stack = env->cfg.insn_stack; -++ int *insn_state = env->cfg.insn_state; -++ -+ if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) -+ return 0; -+ -+@@ -1467,30 +6495,35 @@ static int push_insn(int t, int w, int e -+ return 0; -+ -+ if (w < 0 || w >= env->prog->len) { -+- verbose("jump out of range from insn %d to %d\n", t, w); -++ verbose_linfo(env, t, "%d: ", t); -++ verbose(env, "jump out of range from insn %d to %d\n", t, w); -+ return -EINVAL; -+ } -+ -+ if (e == BRANCH) -+ /* mark branch target for state pruning */ -+- env->explored_states[w] = STATE_LIST_MARK; -++ init_explored_state(env, w); -+ -+ if (insn_state[w] == 0) { -+ /* tree-edge */ -+ insn_state[t] = DISCOVERED | e; -+ insn_state[w] = DISCOVERED; -+- if (cur_stack >= env->prog->len) -++ if (env->cfg.cur_stack >= env->prog->len) -+ return -E2BIG; -+- insn_stack[cur_stack++] = w; -++ insn_stack[env->cfg.cur_stack++] = w; -+ return 1; -+ } else if ((insn_state[w] & 0xF0) == DISCOVERED) { -+- verbose("back-edge from insn %d to %d\n", t, w); -++ if (loop_ok && env->allow_ptr_leaks) -++ return 0; -++ verbose_linfo(env, t, "%d: ", t); -++ verbose_linfo(env, w, "%d: ", w); -++ verbose(env, "back-edge from insn %d to %d\n", t, w); -+ return -EINVAL; -+ } else if (insn_state[w] == EXPLORED) { -+ /* forward- or cross-edge */ -+ insn_state[t] = DISCOVERED | e; -+ } else { -+- verbose("insn state internal bug\n"); -++ verbose(env, "insn state internal bug\n"); -+ return -EFAULT; -+ } -+ return 0; -+@@ -1499,43 +6532,56 @@ static int push_insn(int t, int w, int e -+ /* non-recursive depth-first-search to detect loops in BPF program -+ * loop == back-edge in directed graph -+ */ -+-static int check_cfg(struct verifier_env *env) -++static int check_cfg(struct bpf_verifier_env *env) -+ { -+ struct bpf_insn *insns = env->prog->insnsi; -+ int insn_cnt = env->prog->len; -++ int *insn_stack, *insn_state; -+ int ret = 0; -+ int i, t; -+ -+- insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); -++ insn_state = env->cfg.insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); -+ if (!insn_state) -+ return -ENOMEM; -+ -+- insn_stack = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); -++ insn_stack = env->cfg.insn_stack = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL); -+ if (!insn_stack) { -+- kfree(insn_state); -++ kvfree(insn_state); -+ return -ENOMEM; -+ } -+ -+ insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ -+ insn_stack[0] = 0; /* 0 is the first instruction */ -+- cur_stack = 1; -++ env->cfg.cur_stack = 1; -+ -+ peek_stack: -+- if (cur_stack == 0) -++ if (env->cfg.cur_stack == 0) -+ goto check_state; -+- t = insn_stack[cur_stack - 1]; -++ t = insn_stack[env->cfg.cur_stack - 1]; -+ -+- if (BPF_CLASS(insns[t].code) == BPF_JMP) { -++ if (BPF_CLASS(insns[t].code) == BPF_JMP || -++ BPF_CLASS(insns[t].code) == BPF_JMP32) { -+ u8 opcode = BPF_OP(insns[t].code); -+ -+ if (opcode == BPF_EXIT) { -+ goto mark_explored; -+ } else if (opcode == BPF_CALL) { -+- ret = push_insn(t, t + 1, FALLTHROUGH, env); -++ ret = push_insn(t, t + 1, FALLTHROUGH, env, false); -+ if (ret == 1) -+ goto peek_stack; -+ else if (ret < 0) -+ goto err_free; -++ if (t + 1 < insn_cnt) -++ init_explored_state(env, t + 1); -++ if (insns[t].src_reg == BPF_PSEUDO_CALL) { -++ init_explored_state(env, t); -++ ret = push_insn(t, t + insns[t].imm + 1, BRANCH, -++ env, false); -++ if (ret == 1) -++ goto peek_stack; -++ else if (ret < 0) -++ goto err_free; -++ } -+ } else if (opcode == BPF_JA) { -+ if (BPF_SRC(insns[t].code) != BPF_K) { -+ ret = -EINVAL; -+@@ -1543,25 +6589,31 @@ peek_stack: -+ } -+ /* unconditional jump with single edge */ -+ ret = push_insn(t, t + insns[t].off + 1, -+- FALLTHROUGH, env); -++ FALLTHROUGH, env, true); -+ if (ret == 1) -+ goto peek_stack; -+ else if (ret < 0) -+ goto err_free; -++ /* unconditional jmp is not a good pruning point, -++ * but it's marked, since backtracking needs -++ * to record jmp history in is_state_visited(). -++ */ -++ init_explored_state(env, t + insns[t].off + 1); -+ /* tell verifier to check for equivalent states -+ * after every call and jump -+ */ -+ if (t + 1 < insn_cnt) -+- env->explored_states[t + 1] = STATE_LIST_MARK; -++ init_explored_state(env, t + 1); -+ } else { -+ /* conditional jump with two edges */ -+- ret = push_insn(t, t + 1, FALLTHROUGH, env); -++ init_explored_state(env, t); -++ ret = push_insn(t, t + 1, FALLTHROUGH, env, true); -+ if (ret == 1) -+ goto peek_stack; -+ else if (ret < 0) -+ goto err_free; -+ -+- ret = push_insn(t, t + insns[t].off + 1, BRANCH, env); -++ ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true); -+ if (ret == 1) -+ goto peek_stack; -+ else if (ret < 0) -+@@ -1571,7 +6623,7 @@ peek_stack: -+ /* all other non-branch instructions with single -+ * fall-through edge -+ */ -+- ret = push_insn(t, t + 1, FALLTHROUGH, env); -++ ret = push_insn(t, t + 1, FALLTHROUGH, env, false); -+ if (ret == 1) -+ goto peek_stack; -+ else if (ret < 0) -+@@ -1580,8 +6632,8 @@ peek_stack: -+ -+ mark_explored: -+ insn_state[t] = EXPLORED; -+- if (cur_stack-- <= 0) { -+- verbose("pop stack internal bug\n"); -++ if (env->cfg.cur_stack-- <= 0) { -++ verbose(env, "pop stack internal bug\n"); -+ ret = -EFAULT; -+ goto err_free; -+ } -+@@ -1590,7 +6642,7 @@ mark_explored: -+ check_state: -+ for (i = 0; i < insn_cnt; i++) { -+ if (insn_state[i] != EXPLORED) { -+- verbose("unreachable insn %d\n", i); -++ verbose(env, "unreachable insn %d\n", i); -+ ret = -EINVAL; -+ goto err_free; -+ } -+@@ -1598,11 +6650,616 @@ check_state: -+ ret = 0; /* cfg looks good */ -+ -+ err_free: -+- kfree(insn_state); -+- kfree(insn_stack); -++ kvfree(insn_state); -++ kvfree(insn_stack); -++ env->cfg.insn_state = env->cfg.insn_stack = NULL; -+ return ret; -+ } -+ -++/* The minimum supported BTF func info size */ -++#define MIN_BPF_FUNCINFO_SIZE 8 -++#define MAX_FUNCINFO_REC_SIZE 252 -++ -++static int check_btf_func(struct bpf_verifier_env *env, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ u32 i, nfuncs, urec_size, min_size; -++ u32 krec_size = sizeof(struct bpf_func_info); -++ struct bpf_func_info *krecord; -++ const struct btf_type *type; -++ struct bpf_prog *prog; -++ const struct btf *btf; -++ void __user *urecord; -++ u32 prev_offset = 0; -++ int ret = 0; -++ -++ nfuncs = attr->func_info_cnt; -++ if (!nfuncs) -++ return 0; -++ -++ if (nfuncs != env->subprog_cnt) { -++ verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); -++ return -EINVAL; -++ } -++ -++ urec_size = attr->func_info_rec_size; -++ if (urec_size < MIN_BPF_FUNCINFO_SIZE || -++ urec_size > MAX_FUNCINFO_REC_SIZE || -++ urec_size % sizeof(u32)) { -++ verbose(env, "invalid func info rec size %u\n", urec_size); -++ return -EINVAL; -++ } -++ -++ prog = env->prog; -++ btf = prog->aux->btf; -++ -++ urecord = u64_to_user_ptr(attr->func_info); -++ min_size = min_t(u32, krec_size, urec_size); -++ -++ krecord = kcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); -++ if (!krecord) -++ return -ENOMEM; -++ -++ for (i = 0; i < nfuncs; i++) { -++ ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); -++ if (ret) { -++ if (ret == -E2BIG) { -++ verbose(env, "nonzero tailing record in func info"); -++ /* set the size kernel expects so loader can zero -++ * out the rest of the record. -++ */ -++ if (put_user(min_size, &uattr->func_info_rec_size)) -++ ret = -EFAULT; -++ } -++ goto err_free; -++ } -++ -++ if (copy_from_user(&krecord[i], urecord, min_size)) { -++ ret = -EFAULT; -++ goto err_free; -++ } -++ -++ /* check insn_off */ -++ if (i == 0) { -++ if (krecord[i].insn_off) { -++ verbose(env, -++ "nonzero insn_off %u for the first func info record", -++ krecord[i].insn_off); -++ ret = -EINVAL; -++ goto err_free; -++ } -++ } else if (krecord[i].insn_off <= prev_offset) { -++ verbose(env, -++ "same or smaller insn offset (%u) than previous func info record (%u)", -++ krecord[i].insn_off, prev_offset); -++ ret = -EINVAL; -++ goto err_free; -++ } -++ -++ if (env->subprog_info[i].start != krecord[i].insn_off) { -++ verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); -++ ret = -EINVAL; -++ goto err_free; -++ } -++ -++ /* check type_id */ -++ type = btf_type_by_id(btf, krecord[i].type_id); -++ if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) { -++ verbose(env, "invalid type id %d in func info", -++ krecord[i].type_id); -++ ret = -EINVAL; -++ goto err_free; -++ } -++ -++ prev_offset = krecord[i].insn_off; -++ urecord += urec_size; -++ } -++ -++ prog->aux->func_info = krecord; -++ prog->aux->func_info_cnt = nfuncs; -++ return 0; -++ -++err_free: -++ kvfree(krecord); -++ return ret; -++} -++ -++static void adjust_btf_func(struct bpf_verifier_env *env) -++{ -++ int i; -++ -++ if (!env->prog->aux->func_info) -++ return; -++ -++ for (i = 0; i < env->subprog_cnt; i++) -++ env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; -++} -++ -++#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ -++ sizeof(((struct bpf_line_info *)(0))->line_col)) -++#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE -++ -++static int check_btf_line(struct bpf_verifier_env *env, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; -++ struct bpf_subprog_info *sub; -++ struct bpf_line_info *linfo; -++ struct bpf_prog *prog; -++ const struct btf *btf; -++ void __user *ulinfo; -++ int err; -++ -++ nr_linfo = attr->line_info_cnt; -++ if (!nr_linfo) -++ return 0; -++ -++ rec_size = attr->line_info_rec_size; -++ if (rec_size < MIN_BPF_LINEINFO_SIZE || -++ rec_size > MAX_LINEINFO_REC_SIZE || -++ rec_size & (sizeof(u32) - 1)) -++ return -EINVAL; -++ -++ /* Need to zero it in case the userspace may -++ * pass in a smaller bpf_line_info object. -++ */ -++ linfo = kcalloc(nr_linfo, sizeof(struct bpf_line_info), -++ GFP_KERNEL | __GFP_NOWARN); -++ if (!linfo) -++ return -ENOMEM; -++ -++ prog = env->prog; -++ btf = prog->aux->btf; -++ -++ s = 0; -++ sub = env->subprog_info; -++ ulinfo = u64_to_user_ptr(attr->line_info); -++ expected_size = sizeof(struct bpf_line_info); -++ ncopy = min_t(u32, expected_size, rec_size); -++ for (i = 0; i < nr_linfo; i++) { -++ err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); -++ if (err) { -++ if (err == -E2BIG) { -++ verbose(env, "nonzero tailing record in line_info"); -++ if (put_user(expected_size, -++ &uattr->line_info_rec_size)) -++ err = -EFAULT; -++ } -++ goto err_free; -++ } -++ -++ if (copy_from_user(&linfo[i], ulinfo, ncopy)) { -++ err = -EFAULT; -++ goto err_free; -++ } -++ -++ /* -++ * Check insn_off to ensure -++ * 1) strictly increasing AND -++ * 2) bounded by prog->len -++ * -++ * The linfo[0].insn_off == 0 check logically falls into -++ * the later "missing bpf_line_info for func..." case -++ * because the first linfo[0].insn_off must be the -++ * first sub also and the first sub must have -++ * subprog_info[0].start == 0. -++ */ -++ if ((i && linfo[i].insn_off <= prev_offset) || -++ linfo[i].insn_off >= prog->len) { -++ verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", -++ i, linfo[i].insn_off, prev_offset, -++ prog->len); -++ err = -EINVAL; -++ goto err_free; -++ } -++ -++ if (!prog->insnsi[linfo[i].insn_off].code) { -++ verbose(env, -++ "Invalid insn code at line_info[%u].insn_off\n", -++ i); -++ err = -EINVAL; -++ goto err_free; -++ } -++ -++ if (!btf_name_by_offset(btf, linfo[i].line_off) || -++ !btf_name_by_offset(btf, linfo[i].file_name_off)) { -++ verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); -++ err = -EINVAL; -++ goto err_free; -++ } -++ -++ if (s != env->subprog_cnt) { -++ if (linfo[i].insn_off == sub[s].start) { -++ sub[s].linfo_idx = i; -++ s++; -++ } else if (sub[s].start < linfo[i].insn_off) { -++ verbose(env, "missing bpf_line_info for func#%u\n", s); -++ err = -EINVAL; -++ goto err_free; -++ } -++ } -++ -++ prev_offset = linfo[i].insn_off; -++ ulinfo += rec_size; -++ } -++ -++ if (s != env->subprog_cnt) { -++ verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", -++ env->subprog_cnt - s, s); -++ err = -EINVAL; -++ goto err_free; -++ } -++ -++ prog->aux->linfo = linfo; -++ prog->aux->nr_linfo = nr_linfo; -++ -++ return 0; -++ -++err_free: -++ kvfree(linfo); -++ return err; -++} -++ -++static int check_btf_info(struct bpf_verifier_env *env, -++ const union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ struct btf *btf; -++ int err; -++ -++ if (!attr->func_info_cnt && !attr->line_info_cnt) -++ return 0; -++ -++ btf = btf_get_by_fd(attr->prog_btf_fd); -++ if (IS_ERR(btf)) -++ return PTR_ERR(btf); -++ env->prog->aux->btf = btf; -++ -++ err = check_btf_func(env, attr, uattr); -++ if (err) -++ return err; -++ -++ err = check_btf_line(env, attr, uattr); -++ if (err) -++ return err; -++ -++ return 0; -++} -++ -++/* check %cur's range satisfies %old's */ -++static bool range_within(struct bpf_reg_state *old, -++ struct bpf_reg_state *cur) -++{ -++ return old->umin_value <= cur->umin_value && -++ old->umax_value >= cur->umax_value && -++ old->smin_value <= cur->smin_value && -++ old->smax_value >= cur->smax_value; -++} -++ -++/* Maximum number of register states that can exist at once */ -++#define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) -++struct idpair { -++ u32 old; -++ u32 cur; -++}; -++ -++/* If in the old state two registers had the same id, then they need to have -++ * the same id in the new state as well. But that id could be different from -++ * the old state, so we need to track the mapping from old to new ids. -++ * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent -++ * regs with old id 5 must also have new id 9 for the new state to be safe. But -++ * regs with a different old id could still have new id 9, we don't care about -++ * that. -++ * So we look through our idmap to see if this old id has been seen before. If -++ * so, we require the new id to match; otherwise, we add the id pair to the map. -++ */ -++static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) -++{ -++ unsigned int i; -++ -++ for (i = 0; i < ID_MAP_SIZE; i++) { -++ if (!idmap[i].old) { -++ /* Reached an empty slot; haven't seen this id before */ -++ idmap[i].old = old_id; -++ idmap[i].cur = cur_id; -++ return true; -++ } -++ if (idmap[i].old == old_id) -++ return idmap[i].cur == cur_id; -++ } -++ /* We ran out of idmap slots, which should be impossible */ -++ WARN_ON_ONCE(1); -++ return false; -++} -++ -++static void clean_func_state(struct bpf_verifier_env *env, -++ struct bpf_func_state *st) -++{ -++ enum bpf_reg_liveness live; -++ int i, j; -++ -++ for (i = 0; i < BPF_REG_FP; i++) { -++ live = st->regs[i].live; -++ /* liveness must not touch this register anymore */ -++ st->regs[i].live |= REG_LIVE_DONE; -++ if (!(live & REG_LIVE_READ)) -++ /* since the register is unused, clear its state -++ * to make further comparison simpler -++ */ -++ __mark_reg_not_init(env, &st->regs[i]); -++ } -++ -++ for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { -++ live = st->stack[i].spilled_ptr.live; -++ /* liveness must not touch this stack slot anymore */ -++ st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; -++ if (!(live & REG_LIVE_READ)) { -++ __mark_reg_not_init(env, &st->stack[i].spilled_ptr); -++ for (j = 0; j < BPF_REG_SIZE; j++) -++ st->stack[i].slot_type[j] = STACK_INVALID; -++ } -++ } -++} -++ -++static void clean_verifier_state(struct bpf_verifier_env *env, -++ struct bpf_verifier_state *st) -++{ -++ int i; -++ -++ if (st->frame[0]->regs[0].live & REG_LIVE_DONE) -++ /* all regs in this state in all frames were already marked */ -++ return; -++ -++ for (i = 0; i <= st->curframe; i++) -++ clean_func_state(env, st->frame[i]); -++} -++ -++/* the parentage chains form a tree. -++ * the verifier states are added to state lists at given insn and -++ * pushed into state stack for future exploration. -++ * when the verifier reaches bpf_exit insn some of the verifer states -++ * stored in the state lists have their final liveness state already, -++ * but a lot of states will get revised from liveness point of view when -++ * the verifier explores other branches. -++ * Example: -++ * 1: r0 = 1 -++ * 2: if r1 == 100 goto pc+1 -++ * 3: r0 = 2 -++ * 4: exit -++ * when the verifier reaches exit insn the register r0 in the state list of -++ * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch -++ * of insn 2 and goes exploring further. At the insn 4 it will walk the -++ * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. -++ * -++ * Since the verifier pushes the branch states as it sees them while exploring -++ * the program the condition of walking the branch instruction for the second -++ * time means that all states below this branch were already explored and -++ * their final liveness markes are already propagated. -++ * Hence when the verifier completes the search of state list in is_state_visited() -++ * we can call this clean_live_states() function to mark all liveness states -++ * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' -++ * will not be used. -++ * This function also clears the registers and stack for states that !READ -++ * to simplify state merging. -++ * -++ * Important note here that walking the same branch instruction in the callee -++ * doesn't meant that the states are DONE. The verifier has to compare -++ * the callsites -++ */ -++static void clean_live_states(struct bpf_verifier_env *env, int insn, -++ struct bpf_verifier_state *cur) -++{ -++ struct bpf_verifier_state_list *sl; -++ int i; -++ -++ sl = *explored_state(env, insn); -++ while (sl) { -++ if (sl->state.branches) -++ goto next; -++ if (sl->state.insn_idx != insn || -++ sl->state.curframe != cur->curframe) -++ goto next; -++ for (i = 0; i <= cur->curframe; i++) -++ if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) -++ goto next; -++ clean_verifier_state(env, &sl->state); -++next: -++ sl = sl->next; -++ } -++} -++ -++/* Returns true if (rold safe implies rcur safe) */ -++static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, -++ struct idpair *idmap) -++{ -++ bool equal; -++ -++ if (!(rold->live & REG_LIVE_READ)) -++ /* explored state didn't use this */ -++ return true; -++ -++ equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; -++ -++ if (rold->type == PTR_TO_STACK) -++ /* two stack pointers are equal only if they're pointing to -++ * the same stack frame, since fp-8 in foo != fp-8 in bar -++ */ -++ return equal && rold->frameno == rcur->frameno; -++ -++ if (equal) -++ return true; -++ -++ if (rold->type == NOT_INIT) -++ /* explored state can't have used this */ -++ return true; -++ if (rcur->type == NOT_INIT) -++ return false; -++ switch (rold->type) { -++ case SCALAR_VALUE: -++ if (rcur->type == SCALAR_VALUE) { -++ if (!rold->precise && !rcur->precise) -++ return true; -++ /* new val must satisfy old val knowledge */ -++ return range_within(rold, rcur) && -++ tnum_in(rold->var_off, rcur->var_off); -++ } else { -++ /* We're trying to use a pointer in place of a scalar. -++ * Even if the scalar was unbounded, this could lead to -++ * pointer leaks because scalars are allowed to leak -++ * while pointers are not. We could make this safe in -++ * special cases if root is calling us, but it's -++ * probably not worth the hassle. -++ */ -++ return false; -++ } -++ case PTR_TO_MAP_VALUE: -++ /* If the new min/max/var_off satisfy the old ones and -++ * everything else matches, we are OK. -++ * 'id' is not compared, since it's only used for maps with -++ * bpf_spin_lock inside map element and in such cases if -++ * the rest of the prog is valid for one map element then -++ * it's valid for all map elements regardless of the key -++ * used in bpf_map_lookup() -++ */ -++ return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && -++ range_within(rold, rcur) && -++ tnum_in(rold->var_off, rcur->var_off); -++ case PTR_TO_MAP_VALUE_OR_NULL: -++ /* a PTR_TO_MAP_VALUE could be safe to use as a -++ * PTR_TO_MAP_VALUE_OR_NULL into the same map. -++ * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- -++ * checked, doing so could have affected others with the same -++ * id, and we can't check for that because we lost the id when -++ * we converted to a PTR_TO_MAP_VALUE. -++ */ -++ if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) -++ return false; -++ if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) -++ return false; -++ /* Check our ids match any regs they're supposed to */ -++ return check_ids(rold->id, rcur->id, idmap); -++ case PTR_TO_PACKET_META: -++ case PTR_TO_PACKET: -++ if (rcur->type != rold->type) -++ return false; -++ /* We must have at least as much range as the old ptr -++ * did, so that any accesses which were safe before are -++ * still safe. This is true even if old range < old off, -++ * since someone could have accessed through (ptr - k), or -++ * even done ptr -= k in a register, to get a safe access. -++ */ -++ if (rold->range > rcur->range) -++ return false; -++ /* If the offsets don't match, we can't trust our alignment; -++ * nor can we be sure that we won't fall out of range. -++ */ -++ if (rold->off != rcur->off) -++ return false; -++ /* id relations must be preserved */ -++ if (rold->id && !check_ids(rold->id, rcur->id, idmap)) -++ return false; -++ /* new val must satisfy old val knowledge */ -++ return range_within(rold, rcur) && -++ tnum_in(rold->var_off, rcur->var_off); -++ case PTR_TO_CTX: -++ case CONST_PTR_TO_MAP: -++ case PTR_TO_PACKET_END: -++ case PTR_TO_FLOW_KEYS: -++ case PTR_TO_SOCKET: -++ case PTR_TO_SOCKET_OR_NULL: -++ case PTR_TO_SOCK_COMMON: -++ case PTR_TO_SOCK_COMMON_OR_NULL: -++ case PTR_TO_TCP_SOCK: -++ case PTR_TO_TCP_SOCK_OR_NULL: -++ case PTR_TO_XDP_SOCK: -++ /* Only valid matches are exact, which memcmp() above -++ * would have accepted -++ */ -++ default: -++ /* Don't know what's going on, just say it's not safe */ -++ return false; -++ } -++ -++ /* Shouldn't get here; if we do, say it's not safe */ -++ WARN_ON_ONCE(1); -++ return false; -++} -++ -++static bool stacksafe(struct bpf_func_state *old, -++ struct bpf_func_state *cur, -++ struct idpair *idmap) -++{ -++ int i, spi; -++ -++ /* walk slots of the explored stack and ignore any additional -++ * slots in the current stack, since explored(safe) state -++ * didn't use them -++ */ -++ for (i = 0; i < old->allocated_stack; i++) { -++ spi = i / BPF_REG_SIZE; -++ -++ if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { -++ i += BPF_REG_SIZE - 1; -++ /* explored state didn't use this */ -++ continue; -++ } -++ -++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) -++ continue; -++ -++ /* explored stack has more populated slots than current stack -++ * and these slots were used -++ */ -++ if (i >= cur->allocated_stack) -++ return false; -++ -++ /* if old state was safe with misc data in the stack -++ * it will be safe with zero-initialized stack. -++ * The opposite is not true -++ */ -++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && -++ cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) -++ continue; -++ if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != -++ cur->stack[spi].slot_type[i % BPF_REG_SIZE]) -++ /* Ex: old explored (safe) state has STACK_SPILL in -++ * this stack slot, but current has has STACK_MISC -> -++ * this verifier states are not equivalent, -++ * return false to continue verification of this path -++ */ -++ return false; -++ if (i % BPF_REG_SIZE) -++ continue; -++ if (old->stack[spi].slot_type[0] != STACK_SPILL) -++ continue; -++ if (!regsafe(&old->stack[spi].spilled_ptr, -++ &cur->stack[spi].spilled_ptr, -++ idmap)) -++ /* when explored and current stack slot are both storing -++ * spilled registers, check that stored pointers types -++ * are the same as well. -++ * Ex: explored safe path could have stored -++ * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} -++ * but current path has stored: -++ * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} -++ * such verifier states are not equivalent. -++ * return false to continue verification of this path -++ */ -++ return false; -++ } -++ return true; -++} -++ -++static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) -++{ -++ if (old->acquired_refs != cur->acquired_refs) -++ return false; -++ return !memcmp(old->refs, cur->refs, -++ sizeof(*old->refs) * old->acquired_refs); -++} -++ -+ /* compare two verifier states -+ * -+ * all states stored in state_list are known to be valid, since -+@@ -1629,165 +7286,562 @@ err_free: -+ * whereas register type in current state is meaningful, it means that -+ * the current state will reach 'bpf_exit' instruction safely -+ */ -+-static bool states_equal(struct verifier_state *old, struct verifier_state *cur) -++static bool func_states_equal(struct bpf_func_state *old, -++ struct bpf_func_state *cur) -+ { -++ struct idpair *idmap; -++ bool ret = false; -+ int i; -+ -++ idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); -++ /* If we failed to allocate the idmap, just say it's not safe */ -++ if (!idmap) -++ return false; -++ -+ for (i = 0; i < MAX_BPF_REG; i++) { -+- if (memcmp(&old->regs[i], &cur->regs[i], -+- sizeof(old->regs[0])) != 0) { -+- if (old->regs[i].type == NOT_INIT || -+- (old->regs[i].type == UNKNOWN_VALUE && -+- cur->regs[i].type != NOT_INIT)) -+- continue; -++ if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) -++ goto out_free; -++ } -++ -++ if (!stacksafe(old, cur, idmap)) -++ goto out_free; -++ -++ if (!refsafe(old, cur)) -++ goto out_free; -++ ret = true; -++out_free: -++ kfree(idmap); -++ return ret; -++} -++ -++static bool states_equal(struct bpf_verifier_env *env, -++ struct bpf_verifier_state *old, -++ struct bpf_verifier_state *cur) -++{ -++ int i; -++ -++ if (old->curframe != cur->curframe) -++ return false; -++ -++ /* Verification state from speculative execution simulation -++ * must never prune a non-speculative execution one. -++ */ -++ if (old->speculative && !cur->speculative) -++ return false; -++ -++ if (old->active_spin_lock != cur->active_spin_lock) -++ return false; -++ -++ /* for states to be equal callsites have to be the same -++ * and all frame states need to be equivalent -++ */ -++ for (i = 0; i <= old->curframe; i++) { -++ if (old->frame[i]->callsite != cur->frame[i]->callsite) -++ return false; -++ if (!func_states_equal(old->frame[i], cur->frame[i])) -+ return false; -++ } -++ return true; -++} -++ -++/* Return 0 if no propagation happened. Return negative error code if error -++ * happened. Otherwise, return the propagated bit. -++ */ -++static int propagate_liveness_reg(struct bpf_verifier_env *env, -++ struct bpf_reg_state *reg, -++ struct bpf_reg_state *parent_reg) -++{ -++ u8 parent_flag = parent_reg->live & REG_LIVE_READ; -++ u8 flag = reg->live & REG_LIVE_READ; -++ int err; -++ -++ /* When comes here, read flags of PARENT_REG or REG could be any of -++ * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need -++ * of propagation if PARENT_REG has strongest REG_LIVE_READ64. -++ */ -++ if (parent_flag == REG_LIVE_READ64 || -++ /* Or if there is no read flag from REG. */ -++ !flag || -++ /* Or if the read flag from REG is the same as PARENT_REG. */ -++ parent_flag == flag) -++ return 0; -++ -++ err = mark_reg_read(env, reg, parent_reg, flag); -++ if (err) -++ return err; -++ -++ return flag; -++} -++ -++/* A write screens off any subsequent reads; but write marks come from the -++ * straight-line code between a state and its parent. When we arrive at an -++ * equivalent state (jump target or such) we didn't arrive by the straight-line -++ * code, so read marks in the state must propagate to the parent regardless -++ * of the state's write marks. That's what 'parent == state->parent' comparison -++ * in mark_reg_read() is for. -++ */ -++static int propagate_liveness(struct bpf_verifier_env *env, -++ const struct bpf_verifier_state *vstate, -++ struct bpf_verifier_state *vparent) -++{ -++ struct bpf_reg_state *state_reg, *parent_reg; -++ struct bpf_func_state *state, *parent; -++ int i, frame, err = 0; -++ -++ if (vparent->curframe != vstate->curframe) { -++ WARN(1, "propagate_live: parent frame %d current frame %d\n", -++ vparent->curframe, vstate->curframe); -++ return -EFAULT; -++ } -++ /* Propagate read liveness of registers... */ -++ BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); -++ for (frame = 0; frame <= vstate->curframe; frame++) { -++ parent = vparent->frame[frame]; -++ state = vstate->frame[frame]; -++ parent_reg = parent->regs; -++ state_reg = state->regs; -++ /* We don't need to worry about FP liveness, it's read-only */ -++ for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { -++ err = propagate_liveness_reg(env, &state_reg[i], -++ &parent_reg[i]); -++ if (err < 0) -++ return err; -++ if (err == REG_LIVE_READ64) -++ mark_insn_zext(env, &parent_reg[i]); -++ } -++ -++ /* Propagate stack slots. */ -++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && -++ i < parent->allocated_stack / BPF_REG_SIZE; i++) { -++ parent_reg = &parent->stack[i].spilled_ptr; -++ state_reg = &state->stack[i].spilled_ptr; -++ err = propagate_liveness_reg(env, state_reg, -++ parent_reg); -++ if (err < 0) -++ return err; -+ } -+ } -++ return 0; -++} -+ -+- for (i = 0; i < MAX_BPF_STACK; i++) { -+- if (old->stack_slot_type[i] == STACK_INVALID) -++/* find precise scalars in the previous equivalent state and -++ * propagate them into the current state -++ */ -++static int propagate_precision(struct bpf_verifier_env *env, -++ const struct bpf_verifier_state *old) -++{ -++ struct bpf_reg_state *state_reg; -++ struct bpf_func_state *state; -++ int i, err = 0; -++ -++ state = old->frame[old->curframe]; -++ state_reg = state->regs; -++ for (i = 0; i < BPF_REG_FP; i++, state_reg++) { -++ if (state_reg->type != SCALAR_VALUE || -++ !state_reg->precise) -+ continue; -+- if (old->stack_slot_type[i] != cur->stack_slot_type[i]) -+- /* Ex: old explored (safe) state has STACK_SPILL in -+- * this stack slot, but current has has STACK_MISC -> -+- * this verifier states are not equivalent, -+- * return false to continue verification of this path -+- */ -+- return false; -+- if (i % BPF_REG_SIZE) -++ if (env->log.level & BPF_LOG_LEVEL2) -++ verbose(env, "propagating r%d\n", i); -++ err = mark_chain_precision(env, i); -++ if (err < 0) -++ return err; -++ } -++ -++ for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { -++ if (state->stack[i].slot_type[0] != STACK_SPILL) -+ continue; -+- if (memcmp(&old->spilled_regs[i / BPF_REG_SIZE], -+- &cur->spilled_regs[i / BPF_REG_SIZE], -+- sizeof(old->spilled_regs[0]))) -+- /* when explored and current stack slot types are -+- * the same, check that stored pointers types -+- * are the same as well. -+- * Ex: explored safe path could have stored -+- * (struct reg_state) {.type = PTR_TO_STACK, .imm = -8} -+- * but current path has stored: -+- * (struct reg_state) {.type = PTR_TO_STACK, .imm = -16} -+- * such verifier states are not equivalent. -+- * return false to continue verification of this path -+- */ -+- return false; -+- else -++ state_reg = &state->stack[i].spilled_ptr; -++ if (state_reg->type != SCALAR_VALUE || -++ !state_reg->precise) -+ continue; -++ if (env->log.level & BPF_LOG_LEVEL2) -++ verbose(env, "propagating fp%d\n", -++ (-i - 1) * BPF_REG_SIZE); -++ err = mark_chain_precision_stack(env, i); -++ if (err < 0) -++ return err; -+ } -++ return 0; -++} -++ -++static bool states_maybe_looping(struct bpf_verifier_state *old, -++ struct bpf_verifier_state *cur) -++{ -++ struct bpf_func_state *fold, *fcur; -++ int i, fr = cur->curframe; -++ -++ if (old->curframe != fr) -++ return false; -++ -++ fold = old->frame[fr]; -++ fcur = cur->frame[fr]; -++ for (i = 0; i < MAX_BPF_REG; i++) -++ if (memcmp(&fold->regs[i], &fcur->regs[i], -++ offsetof(struct bpf_reg_state, parent))) -++ return false; -+ return true; -+ } -+ -+-static int is_state_visited(struct verifier_env *env, int insn_idx) -++ -++static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) -+ { -+- struct verifier_state_list *new_sl; -+- struct verifier_state_list *sl; -++ struct bpf_verifier_state_list *new_sl; -++ struct bpf_verifier_state_list *sl, **pprev; -++ struct bpf_verifier_state *cur = env->cur_state, *new; -++ int i, j, err, states_cnt = 0; -++ bool add_new_state = env->test_state_freq ? true : false; -+ -+- sl = env->explored_states[insn_idx]; -+- if (!sl) -++ cur->last_insn_idx = env->prev_insn_idx; -++ if (!env->insn_aux_data[insn_idx].prune_point) -+ /* this 'insn_idx' instruction wasn't marked, so we will not -+ * be doing state search here -+ */ -+ return 0; -+ -+- while (sl != STATE_LIST_MARK) { -+- if (states_equal(&sl->state, &env->cur_state)) -++ /* bpf progs typically have pruning point every 4 instructions -++ * http://vger.kernel.org/bpfconf2019.html#session-1 -++ * Do not add new state for future pruning if the verifier hasn't seen -++ * at least 2 jumps and at least 8 instructions. -++ * This heuristics helps decrease 'total_states' and 'peak_states' metric. -++ * In tests that amounts to up to 50% reduction into total verifier -++ * memory consumption and 20% verifier time speedup. -++ */ -++ if (env->jmps_processed - env->prev_jmps_processed >= 2 && -++ env->insn_processed - env->prev_insn_processed >= 8) -++ add_new_state = true; -++ -++ pprev = explored_state(env, insn_idx); -++ sl = *pprev; -++ -++ clean_live_states(env, insn_idx, cur); -++ -++ while (sl) { -++ states_cnt++; -++ if (sl->state.insn_idx != insn_idx) -++ goto next; -++ if (sl->state.branches) { -++ if (states_maybe_looping(&sl->state, cur) && -++ states_equal(env, &sl->state, cur)) { -++ verbose_linfo(env, insn_idx, "; "); -++ verbose(env, "infinite loop detected at insn %d\n", insn_idx); -++ return -EINVAL; -++ } -++ /* if the verifier is processing a loop, avoid adding new state -++ * too often, since different loop iterations have distinct -++ * states and may not help future pruning. -++ * This threshold shouldn't be too low to make sure that -++ * a loop with large bound will be rejected quickly. -++ * The most abusive loop will be: -++ * r1 += 1 -++ * if r1 < 1000000 goto pc-2 -++ * 1M insn_procssed limit / 100 == 10k peak states. -++ * This threshold shouldn't be too high either, since states -++ * at the end of the loop are likely to be useful in pruning. -++ */ -++ if (env->jmps_processed - env->prev_jmps_processed < 20 && -++ env->insn_processed - env->prev_insn_processed < 100) -++ add_new_state = false; -++ goto miss; -++ } -++ if (states_equal(env, &sl->state, cur)) { -++ sl->hit_cnt++; -+ /* reached equivalent register/stack state, -+- * prune the search -++ * prune the search. -++ * Registers read by the continuation are read by us. -++ * If we have any write marks in env->cur_state, they -++ * will prevent corresponding reads in the continuation -++ * from reaching our parent (an explored_state). Our -++ * own state will get the read marks recorded, but -++ * they'll be immediately forgotten as we're pruning -++ * this state and will pop a new one. -+ */ -+- return 1; -+- sl = sl->next; -+- } -++ err = propagate_liveness(env, &sl->state, cur); -+ -+- /* there were no equivalent states, remember current one. -+- * technically the current state is not proven to be safe yet, -+- * but it will either reach bpf_exit (which means it's safe) or -+- * it will be rejected. Since there are no loops, we won't be -+- * seeing this 'insn_idx' instruction again on the way to bpf_exit -++ /* if previous state reached the exit with precision and -++ * current state is equivalent to it (except precsion marks) -++ * the precision needs to be propagated back in -++ * the current state. -++ */ -++ err = err ? : push_jmp_history(env, cur); -++ err = err ? : propagate_precision(env, &sl->state); -++ if (err) -++ return err; -++ return 1; -++ } -++miss: -++ /* when new state is not going to be added do not increase miss count. -++ * Otherwise several loop iterations will remove the state -++ * recorded earlier. The goal of these heuristics is to have -++ * states from some iterations of the loop (some in the beginning -++ * and some at the end) to help pruning. -++ */ -++ if (add_new_state) -++ sl->miss_cnt++; -++ /* heuristic to determine whether this state is beneficial -++ * to keep checking from state equivalence point of view. -++ * Higher numbers increase max_states_per_insn and verification time, -++ * but do not meaningfully decrease insn_processed. -++ */ -++ if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { -++ /* the state is unlikely to be useful. Remove it to -++ * speed up verification -++ */ -++ *pprev = sl->next; -++ if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { -++ u32 br = sl->state.branches; -++ -++ WARN_ONCE(br, -++ "BUG live_done but branches_to_explore %d\n", -++ br); -++ free_verifier_state(&sl->state, false); -++ kfree(sl); -++ env->peak_states--; -++ } else { -++ /* cannot free this state, since parentage chain may -++ * walk it later. Add it for free_list instead to -++ * be freed at the end of verification -++ */ -++ sl->next = env->free_list; -++ env->free_list = sl; -++ } -++ sl = *pprev; -++ continue; -++ } -++next: -++ pprev = &sl->next; -++ sl = *pprev; -++ } -++ -++ if (env->max_states_per_insn < states_cnt) -++ env->max_states_per_insn = states_cnt; -++ -++ if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) -++ return push_jmp_history(env, cur); -++ -++ if (!add_new_state) -++ return push_jmp_history(env, cur); -++ -++ /* There were no equivalent states, remember the current one. -++ * Technically the current state is not proven to be safe yet, -++ * but it will either reach outer most bpf_exit (which means it's safe) -++ * or it will be rejected. When there are no loops the verifier won't be -++ * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) -++ * again on the way to bpf_exit. -++ * When looping the sl->state.branches will be > 0 and this state -++ * will not be considered for equivalence until branches == 0. -+ */ -+- new_sl = kmalloc(sizeof(struct verifier_state_list), GFP_USER); -++ new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); -+ if (!new_sl) -+ return -ENOMEM; -++ env->total_states++; -++ env->peak_states++; -++ env->prev_jmps_processed = env->jmps_processed; -++ env->prev_insn_processed = env->insn_processed; -+ -+ /* add new state to the head of linked list */ -+- memcpy(&new_sl->state, &env->cur_state, sizeof(env->cur_state)); -+- new_sl->next = env->explored_states[insn_idx]; -+- env->explored_states[insn_idx] = new_sl; -++ new = &new_sl->state; -++ err = copy_verifier_state(new, cur); -++ if (err) { -++ free_verifier_state(new, false); -++ kfree(new_sl); -++ return err; -++ } -++ new->insn_idx = insn_idx; -++ WARN_ONCE(new->branches != 1, -++ "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); -++ -++ cur->parent = new; -++ cur->first_insn_idx = insn_idx; -++ clear_jmp_history(cur); -++ new_sl->next = *explored_state(env, insn_idx); -++ *explored_state(env, insn_idx) = new_sl; -++ /* connect new state to parentage chain. Current frame needs all -++ * registers connected. Only r6 - r9 of the callers are alive (pushed -++ * to the stack implicitly by JITs) so in callers' frames connect just -++ * r6 - r9 as an optimization. Callers will have r1 - r5 connected to -++ * the state of the call instruction (with WRITTEN set), and r0 comes -++ * from callee with its full parentage chain, anyway. -++ */ -++ /* clear write marks in current state: the writes we did are not writes -++ * our child did, so they don't screen off its reads from us. -++ * (There are no read marks in current state, because reads always mark -++ * their parent and current state never has children yet. Only -++ * explored_states can get read marks.) -++ */ -++ for (j = 0; j <= cur->curframe; j++) { -++ for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) -++ cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; -++ for (i = 0; i < BPF_REG_FP; i++) -++ cur->frame[j]->regs[i].live = REG_LIVE_NONE; -++ } -++ -++ /* all stack frames are accessible from callee, clear them all */ -++ for (j = 0; j <= cur->curframe; j++) { -++ struct bpf_func_state *frame = cur->frame[j]; -++ struct bpf_func_state *newframe = new->frame[j]; -++ -++ for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { -++ frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; -++ frame->stack[i].spilled_ptr.parent = -++ &newframe->stack[i].spilled_ptr; -++ } -++ } -+ return 0; -+ } -+ -+-static int do_check(struct verifier_env *env) -++/* Return true if it's OK to have the same insn return a different type. */ -++static bool reg_type_mismatch_ok(enum bpf_reg_type type) -+ { -+- struct verifier_state *state = &env->cur_state; -++ switch (type) { -++ case PTR_TO_CTX: -++ case PTR_TO_SOCKET: -++ case PTR_TO_SOCKET_OR_NULL: -++ case PTR_TO_SOCK_COMMON: -++ case PTR_TO_SOCK_COMMON_OR_NULL: -++ case PTR_TO_TCP_SOCK: -++ case PTR_TO_TCP_SOCK_OR_NULL: -++ case PTR_TO_XDP_SOCK: -++ return false; -++ default: -++ return true; -++ } -++} -++ -++/* If an instruction was previously used with particular pointer types, then we -++ * need to be careful to avoid cases such as the below, where it may be ok -++ * for one branch accessing the pointer, but not ok for the other branch: -++ * -++ * R1 = sock_ptr -++ * goto X; -++ * ... -++ * R1 = some_other_valid_ptr; -++ * goto X; -++ * ... -++ * R2 = *(u32 *)(R1 + 0); -++ */ -++static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) -++{ -++ return src != prev && (!reg_type_mismatch_ok(src) || -++ !reg_type_mismatch_ok(prev)); -++} -++ -++static int do_check(struct bpf_verifier_env *env) -++{ -++ struct bpf_verifier_state *state; -+ struct bpf_insn *insns = env->prog->insnsi; -+- struct reg_state *regs = state->regs; -++ struct bpf_reg_state *regs; -+ int insn_cnt = env->prog->len; -+- int insn_idx, prev_insn_idx = 0; -+- int insn_processed = 0; -+ bool do_print_state = false; -++ int prev_insn_idx = -1; -++ -++ env->prev_linfo = NULL; -++ -++ state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); -++ if (!state) -++ return -ENOMEM; -++ state->curframe = 0; -++ state->speculative = false; -++ state->branches = 1; -++ state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); -++ if (!state->frame[0]) { -++ kfree(state); -++ return -ENOMEM; -++ } -++ env->cur_state = state; -++ init_func_state(env, state->frame[0], -++ BPF_MAIN_FUNC /* callsite */, -++ 0 /* frameno */, -++ 0 /* subprogno, zero == main subprog */); -+ -+- init_reg_state(regs); -+- insn_idx = 0; -+ for (;;) { -+ struct bpf_insn *insn; -+ u8 class; -+ int err; -+ -+- if (insn_idx >= insn_cnt) { -+- verbose("invalid insn idx %d insn_cnt %d\n", -+- insn_idx, insn_cnt); -++ env->prev_insn_idx = prev_insn_idx; -++ if (env->insn_idx >= insn_cnt) { -++ verbose(env, "invalid insn idx %d insn_cnt %d\n", -++ env->insn_idx, insn_cnt); -+ return -EFAULT; -+ } -+ -+- insn = &insns[insn_idx]; -++ insn = &insns[env->insn_idx]; -+ class = BPF_CLASS(insn->code); -+ -+- if (++insn_processed > 32768) { -+- verbose("BPF program is too large. Proccessed %d insn\n", -+- insn_processed); -++ if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { -++ verbose(env, -++ "BPF program is too large. Processed %d insn\n", -++ env->insn_processed); -+ return -E2BIG; -+ } -+ -+- err = is_state_visited(env, insn_idx); -++ err = is_state_visited(env, env->insn_idx); -+ if (err < 0) -+ return err; -+ if (err == 1) { -+ /* found equivalent state, can prune the search */ -+- if (log_level) { -++ if (env->log.level & BPF_LOG_LEVEL) { -+ if (do_print_state) -+- verbose("\nfrom %d to %d: safe\n", -+- prev_insn_idx, insn_idx); -++ verbose(env, "\nfrom %d to %d%s: safe\n", -++ env->prev_insn_idx, env->insn_idx, -++ env->cur_state->speculative ? -++ " (speculative execution)" : ""); -+ else -+- verbose("%d: safe\n", insn_idx); -++ verbose(env, "%d: safe\n", env->insn_idx); -+ } -+ goto process_bpf_exit; -+ } -+ -+- if (log_level && do_print_state) { -+- verbose("\nfrom %d to %d:", prev_insn_idx, insn_idx); -+- print_verifier_state(env); -++ if (signal_pending(current)) -++ return -EAGAIN; -++ -++ if (need_resched()) -++ cond_resched(); -++ -++ if (env->log.level & BPF_LOG_LEVEL2 || -++ (env->log.level & BPF_LOG_LEVEL && do_print_state)) { -++ if (env->log.level & BPF_LOG_LEVEL2) -++ verbose(env, "%d:", env->insn_idx); -++ else -++ verbose(env, "\nfrom %d to %d%s:", -++ env->prev_insn_idx, env->insn_idx, -++ env->cur_state->speculative ? -++ " (speculative execution)" : ""); -++ print_verifier_state(env, state->frame[state->curframe]); -+ do_print_state = false; -+ } -+ -+- if (log_level) { -+- verbose("%d: ", insn_idx); -+- print_bpf_insn(env, insn); -++ if (env->log.level & BPF_LOG_LEVEL) { -++ const struct bpf_insn_cbs cbs = { -++ .cb_print = verbose, -++ .private_data = env, -++ }; -++ -++ verbose_linfo(env, env->insn_idx, "; "); -++ verbose(env, "%d: ", env->insn_idx); -++ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); -+ } -+ -++ regs = cur_regs(env); -++ sanitize_mark_insn_seen(env); -++ prev_insn_idx = env->insn_idx; -++ -+ if (class == BPF_ALU || class == BPF_ALU64) { -+ err = check_alu_op(env, insn); -+ if (err) -+ return err; -+ -+ } else if (class == BPF_LDX) { -+- enum bpf_reg_type src_reg_type; -++ enum bpf_reg_type *prev_src_type, src_reg_type; -+ -+ /* check for reserved fields is already done */ -+ -+ /* check src operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ -+- err = check_reg_arg(regs, insn->dst_reg, DST_OP_NO_MARK); -++ err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); -+ if (err) -+ return err; -+ -+@@ -1796,27 +7850,22 @@ static int do_check(struct verifier_env -+ /* check that memory (src_reg + off) is readable, -+ * the state of dst_reg will be updated by this func -+ */ -+- err = check_mem_access(env, insn->src_reg, insn->off, -+- BPF_SIZE(insn->code), BPF_READ, -+- insn->dst_reg); -++ err = check_mem_access(env, env->insn_idx, insn->src_reg, -++ insn->off, BPF_SIZE(insn->code), -++ BPF_READ, insn->dst_reg, false); -+ if (err) -+ return err; -+ -+- if (BPF_SIZE(insn->code) != BPF_W) { -+- insn_idx++; -+- continue; -+- } -++ prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; -+ -+- if (insn->imm == 0) { -++ if (*prev_src_type == NOT_INIT) { -+ /* saw a valid insn -+ * dst_reg = *(u32 *)(src_reg + off) -+- * use reserved 'imm' field to mark this insn -++ * save type to validate intersecting paths -+ */ -+- insn->imm = src_reg_type; -++ *prev_src_type = src_reg_type; -+ -+- } else if (src_reg_type != insn->imm && -+- (src_reg_type == PTR_TO_CTX || -+- insn->imm == PTR_TO_CTX)) { -++ } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { -+ /* ABuser program is trying to use the same insn -+ * dst_reg = *(u32*) (src_reg + off) -+ * with different pointer types: -+@@ -1824,79 +7873,98 @@ static int do_check(struct verifier_env -+ * src_reg == stack|map in some other branch. -+ * Reject it. -+ */ -+- verbose("same insn cannot be used with different pointers\n"); -++ verbose(env, "same insn cannot be used with different pointers\n"); -+ return -EINVAL; -+ } -+ -+ } else if (class == BPF_STX) { -+- enum bpf_reg_type dst_reg_type; -++ enum bpf_reg_type *prev_dst_type, dst_reg_type; -+ -+ if (BPF_MODE(insn->code) == BPF_XADD) { -+- err = check_xadd(env, insn); -++ err = check_xadd(env, env->insn_idx, insn); -+ if (err) -+ return err; -+- insn_idx++; -++ env->insn_idx++; -+ continue; -+ } -+ -+ /* check src1 operand */ -+- err = check_reg_arg(regs, insn->src_reg, SRC_OP); -++ err = check_reg_arg(env, insn->src_reg, SRC_OP); -+ if (err) -+ return err; -+ /* check src2 operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -+ dst_reg_type = regs[insn->dst_reg].type; -+ -+ /* check that memory (dst_reg + off) is writeable */ -+- err = check_mem_access(env, insn->dst_reg, insn->off, -+- BPF_SIZE(insn->code), BPF_WRITE, -+- insn->src_reg); -++ err = check_mem_access(env, env->insn_idx, insn->dst_reg, -++ insn->off, BPF_SIZE(insn->code), -++ BPF_WRITE, insn->src_reg, false); -+ if (err) -+ return err; -+ -+- if (insn->imm == 0) { -+- insn->imm = dst_reg_type; -+- } else if (dst_reg_type != insn->imm && -+- (dst_reg_type == PTR_TO_CTX || -+- insn->imm == PTR_TO_CTX)) { -+- verbose("same insn cannot be used with different pointers\n"); -++ prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; -++ -++ if (*prev_dst_type == NOT_INIT) { -++ *prev_dst_type = dst_reg_type; -++ } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { -++ verbose(env, "same insn cannot be used with different pointers\n"); -+ return -EINVAL; -+ } -+ -+ } else if (class == BPF_ST) { -+ if (BPF_MODE(insn->code) != BPF_MEM || -+ insn->src_reg != BPF_REG_0) { -+- verbose("BPF_ST uses reserved fields\n"); -++ verbose(env, "BPF_ST uses reserved fields\n"); -+ return -EINVAL; -+ } -+ /* check src operand */ -+- err = check_reg_arg(regs, insn->dst_reg, SRC_OP); -++ err = check_reg_arg(env, insn->dst_reg, SRC_OP); -+ if (err) -+ return err; -+ -++ if (is_ctx_reg(env, insn->dst_reg)) { -++ verbose(env, "BPF_ST stores into R%d %s is not allowed\n", -++ insn->dst_reg, -++ reg_type_str[reg_state(env, insn->dst_reg)->type]); -++ return -EACCES; -++ } -++ -+ /* check that memory (dst_reg + off) is writeable */ -+- err = check_mem_access(env, insn->dst_reg, insn->off, -+- BPF_SIZE(insn->code), BPF_WRITE, -+- -1); -++ err = check_mem_access(env, env->insn_idx, insn->dst_reg, -++ insn->off, BPF_SIZE(insn->code), -++ BPF_WRITE, -1, false); -+ if (err) -+ return err; -+ -+- } else if (class == BPF_JMP) { -++ } else if (class == BPF_JMP || class == BPF_JMP32) { -+ u8 opcode = BPF_OP(insn->code); -+ -++ env->jmps_processed++; -+ if (opcode == BPF_CALL) { -+ if (BPF_SRC(insn->code) != BPF_K || -+ insn->off != 0 || -+- insn->src_reg != BPF_REG_0 || -+- insn->dst_reg != BPF_REG_0) { -+- verbose("BPF_CALL uses reserved fields\n"); -++ (insn->src_reg != BPF_REG_0 && -++ insn->src_reg != BPF_PSEUDO_CALL) || -++ insn->dst_reg != BPF_REG_0 || -++ class == BPF_JMP32) { -++ verbose(env, "BPF_CALL uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+- err = check_call(env, insn->imm); -++ if (env->cur_state->active_spin_lock && -++ (insn->src_reg == BPF_PSEUDO_CALL || -++ insn->imm != BPF_FUNC_spin_unlock)) { -++ verbose(env, "function calls are not allowed while holding a lock\n"); -++ return -EINVAL; -++ } -++ if (insn->src_reg == BPF_PSEUDO_CALL) -++ err = check_func_call(env, insn, &env->insn_idx); -++ else -++ err = check_helper_call(env, insn->imm, env->insn_idx); -+ if (err) -+ return err; -+ -+@@ -1904,48 +7972,75 @@ static int do_check(struct verifier_env -+ if (BPF_SRC(insn->code) != BPF_K || -+ insn->imm != 0 || -+ insn->src_reg != BPF_REG_0 || -+- insn->dst_reg != BPF_REG_0) { -+- verbose("BPF_JA uses reserved fields\n"); -++ insn->dst_reg != BPF_REG_0 || -++ class == BPF_JMP32) { -++ verbose(env, "BPF_JA uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+- insn_idx += insn->off + 1; -++ env->insn_idx += insn->off + 1; -+ continue; -+ -+ } else if (opcode == BPF_EXIT) { -+ if (BPF_SRC(insn->code) != BPF_K || -+ insn->imm != 0 || -+ insn->src_reg != BPF_REG_0 || -+- insn->dst_reg != BPF_REG_0) { -+- verbose("BPF_EXIT uses reserved fields\n"); -++ insn->dst_reg != BPF_REG_0 || -++ class == BPF_JMP32) { -++ verbose(env, "BPF_EXIT uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -++ if (env->cur_state->active_spin_lock) { -++ verbose(env, "bpf_spin_unlock is missing\n"); -++ return -EINVAL; -++ } -++ -++ if (state->curframe) { -++ /* exit from nested function */ -++ err = prepare_func_exit(env, &env->insn_idx); -++ if (err) -++ return err; -++ do_print_state = true; -++ continue; -++ } -++ -++ err = check_reference_leak(env); -++ if (err) -++ return err; -++ -+ /* eBPF calling convetion is such that R0 is used -+ * to return the value from eBPF program. -+ * Make sure that it's readable at this time -+ * of bpf_exit, which means that program wrote -+ * something into it earlier -+ */ -+- err = check_reg_arg(regs, BPF_REG_0, SRC_OP); -++ err = check_reg_arg(env, BPF_REG_0, SRC_OP); -+ if (err) -+ return err; -+ -+ if (is_pointer_value(env, BPF_REG_0)) { -+- verbose("R0 leaks addr as return value\n"); -++ verbose(env, "R0 leaks addr as return value\n"); -+ return -EACCES; -+ } -+ -++ err = check_return_code(env); -++ if (err) -++ return err; -+ process_bpf_exit: -+- insn_idx = pop_stack(env, &prev_insn_idx); -+- if (insn_idx < 0) { -++ update_branch_counts(env, env->cur_state); -++ err = pop_stack(env, &prev_insn_idx, -++ &env->insn_idx); -++ if (err < 0) { -++ if (err != -ENOENT) -++ return err; -+ break; -+ } else { -+ do_print_state = true; -+ continue; -+ } -+ } else { -+- err = check_cond_jmp_op(env, insn, &insn_idx); -++ err = check_cond_jmp_op(env, insn, &env->insn_idx); -+ if (err) -+ return err; -+ } -+@@ -1962,83 +8057,194 @@ process_bpf_exit: -+ if (err) -+ return err; -+ -+- insn_idx++; -++ env->insn_idx++; -++ sanitize_mark_insn_seen(env); -+ } else { -+- verbose("invalid BPF_LD mode\n"); -++ verbose(env, "invalid BPF_LD mode\n"); -+ return -EINVAL; -+ } -+ } else { -+- verbose("unknown insn class %d\n", class); -++ verbose(env, "unknown insn class %d\n", class); -+ return -EINVAL; -+ } -+ -+- insn_idx++; -++ env->insn_idx++; -+ } -+ -++ env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; -+ return 0; -+ } -+ -++static int check_map_prealloc(struct bpf_map *map) -++{ -++ return (map->map_type != BPF_MAP_TYPE_HASH && -++ map->map_type != BPF_MAP_TYPE_PERCPU_HASH && -++ map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || -++ !(map->map_flags & BPF_F_NO_PREALLOC); -++} -++ -++static bool is_tracing_prog_type(enum bpf_prog_type type) -++{ -++ switch (type) { -++ case BPF_PROG_TYPE_KPROBE: -++ case BPF_PROG_TYPE_TRACEPOINT: -++ case BPF_PROG_TYPE_PERF_EVENT: -++ case BPF_PROG_TYPE_RAW_TRACEPOINT: -++ return true; -++ default: -++ return false; -++ } -++} -++ -++static int check_map_prog_compatibility(struct bpf_verifier_env *env, -++ struct bpf_map *map, -++ struct bpf_prog *prog) -++ -++{ -++ /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use -++ * preallocated hash maps, since doing memory allocation -++ * in overflow_handler can crash depending on where nmi got -++ * triggered. -++ */ -++ if (prog->type == BPF_PROG_TYPE_PERF_EVENT) { -++ if (!check_map_prealloc(map)) { -++ verbose(env, "perf_event programs can only use preallocated hash map\n"); -++ return -EINVAL; -++ } -++ if (map->inner_map_meta && -++ !check_map_prealloc(map->inner_map_meta)) { -++ verbose(env, "perf_event programs can only use preallocated inner hash map\n"); -++ return -EINVAL; -++ } -++ } -++ -++ if ((is_tracing_prog_type(prog->type) || -++ prog->type == BPF_PROG_TYPE_SOCKET_FILTER) && -++ map_value_has_spin_lock(map)) { -++ verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); -++ return -EINVAL; -++ } -++ -++ return 0; -++} -++ -++static bool bpf_map_is_cgroup_storage(struct bpf_map *map) -++{ -++ return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || -++ map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); -++} -++ -+ /* look for pseudo eBPF instructions that access map FDs and -+ * replace them with actual map pointers -+ */ -+-static int replace_map_fd_with_map_ptr(struct verifier_env *env) -++static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) -+ { -+ struct bpf_insn *insn = env->prog->insnsi; -+ int insn_cnt = env->prog->len; -+- int i, j; -++ int i, j, err; -++ -++ err = bpf_prog_calc_tag(env->prog); -++ if (err) -++ return err; -+ -+ for (i = 0; i < insn_cnt; i++, insn++) { -+ if (BPF_CLASS(insn->code) == BPF_LDX && -+ (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { -+- verbose("BPF_LDX uses reserved fields\n"); -++ verbose(env, "BPF_LDX uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ if (BPF_CLASS(insn->code) == BPF_STX && -+ ((BPF_MODE(insn->code) != BPF_MEM && -+ BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { -+- verbose("BPF_STX uses reserved fields\n"); -++ verbose(env, "BPF_STX uses reserved fields\n"); -+ return -EINVAL; -+ } -+ -+ if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { -++ struct bpf_insn_aux_data *aux; -+ struct bpf_map *map; -+ struct fd f; -++ u64 addr; -+ -+ if (i == insn_cnt - 1 || insn[1].code != 0 || -+ insn[1].dst_reg != 0 || insn[1].src_reg != 0 || -+ insn[1].off != 0) { -+- verbose("invalid bpf_ld_imm64 insn\n"); -++ verbose(env, "invalid bpf_ld_imm64 insn\n"); -+ return -EINVAL; -+ } -+ -+- if (insn->src_reg == 0) -++ if (insn[0].src_reg == 0) -+ /* valid generic load 64-bit imm */ -+ goto next_insn; -+ -+- if (insn->src_reg != BPF_PSEUDO_MAP_FD) { -+- verbose("unrecognized bpf_ld_imm64 insn\n"); -++ /* In final convert_pseudo_ld_imm64() step, this is -++ * converted into regular 64-bit imm load insn. -++ */ -++ if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && -++ insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || -++ (insn[0].src_reg == BPF_PSEUDO_MAP_FD && -++ insn[1].imm != 0)) { -++ verbose(env, -++ "unrecognized bpf_ld_imm64 insn\n"); -+ return -EINVAL; -+ } -+ -+- f = fdget(insn->imm); -++ f = fdget(insn[0].imm); -+ map = __bpf_map_get(f); -+ if (IS_ERR(map)) { -+- verbose("fd %d is not pointing to valid bpf_map\n", -+- insn->imm); -++ verbose(env, "fd %d is not pointing to valid bpf_map\n", -++ insn[0].imm); -+ return PTR_ERR(map); -+ } -+ -+- /* store map pointer inside BPF_LD_IMM64 instruction */ -+- insn[0].imm = (u32) (unsigned long) map; -+- insn[1].imm = ((u64) (unsigned long) map) >> 32; -++ err = check_map_prog_compatibility(env, map, env->prog); -++ if (err) { -++ fdput(f); -++ return err; -++ } -++ -++ aux = &env->insn_aux_data[i]; -++ if (insn->src_reg == BPF_PSEUDO_MAP_FD) { -++ addr = (unsigned long)map; -++ } else { -++ u32 off = insn[1].imm; -++ -++ if (off >= BPF_MAX_VAR_OFF) { -++ verbose(env, "direct value offset of %u is not allowed\n", off); -++ fdput(f); -++ return -EINVAL; -++ } -++ -++ if (!map->ops->map_direct_value_addr) { -++ verbose(env, "no direct value access support for this map type\n"); -++ fdput(f); -++ return -EINVAL; -++ } -++ -++ err = map->ops->map_direct_value_addr(map, &addr, off); -++ if (err) { -++ verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", -++ map->value_size, off); -++ fdput(f); -++ return err; -++ } -++ -++ aux->map_off = off; -++ addr += off; -++ } -++ -++ insn[0].imm = (u32)addr; -++ insn[1].imm = addr >> 32; -+ -+ /* check whether we recorded this map already */ -+- for (j = 0; j < env->used_map_cnt; j++) -++ for (j = 0; j < env->used_map_cnt; j++) { -+ if (env->used_maps[j] == map) { -++ aux->map_index = j; -+ fdput(f); -+ goto next_insn; -+ } -++ } -+ -+ if (env->used_map_cnt >= MAX_USED_MAPS) { -+ fdput(f); -+@@ -2048,19 +8254,31 @@ static int replace_map_fd_with_map_ptr(s -+ /* hold the map. If the program is rejected by verifier, -+ * the map will be released by release_maps() or it -+ * will be used by the valid program until it's unloaded -+- * and all maps are released in free_bpf_prog_info() -++ * and all maps are released in free_used_maps() -+ */ -+ map = bpf_map_inc(map, false); -+ if (IS_ERR(map)) { -+ fdput(f); -+ return PTR_ERR(map); -+ } -++ -++ aux->map_index = env->used_map_cnt; -+ env->used_maps[env->used_map_cnt++] = map; -+ -++ if (bpf_map_is_cgroup_storage(map)) -++ return -EINVAL; -++ -+ fdput(f); -+ next_insn: -+ insn++; -+ i++; -++ continue; -++ } -++ -++ /* Basic sanity check before we invest more work here. */ -++ if (!bpf_opcode_in_insntable(insn->code)) { -++ verbose(env, "unknown opcode %02x\n", insn->code); -++ return -EINVAL; -+ } -+ } -+ -+@@ -2072,7 +8290,7 @@ next_insn: -+ } -+ -+ /* drop refcnt of maps used by the rejected program */ -+-static void release_maps(struct verifier_env *env) -++static void release_maps(struct bpf_verifier_env *env) -+ { -+ int i; -+ -+@@ -2081,7 +8299,7 @@ static void release_maps(struct verifier -+ } -+ -+ /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ -+-static void convert_pseudo_ld_imm64(struct verifier_env *env) -++static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) -+ { -+ struct bpf_insn *insn = env->prog->insnsi; -+ int insn_cnt = env->prog->len; -+@@ -2092,201 +8310,1266 @@ static void convert_pseudo_ld_imm64(stru -+ insn->src_reg = 0; -+ } -+ -+-static void adjust_branches(struct bpf_prog *prog, int pos, int delta) -++/* single env->prog->insni[off] instruction was replaced with the range -++ * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying -++ * [0, off) and [off, end) to new locations, so the patched range stays zero -++ */ -++static int adjust_insn_aux_data(struct bpf_verifier_env *env, -++ struct bpf_prog *new_prog, u32 off, u32 cnt) -+ { -+- struct bpf_insn *insn = prog->insnsi; -+- int insn_cnt = prog->len; -++ struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; -++ struct bpf_insn *insn = new_prog->insnsi; -++ bool old_seen = old_data[off].seen; -++ u32 prog_len; -+ int i; -+ -+- for (i = 0; i < insn_cnt; i++, insn++) { -+- if (BPF_CLASS(insn->code) != BPF_JMP || -+- BPF_OP(insn->code) == BPF_CALL || -+- BPF_OP(insn->code) == BPF_EXIT) -++ /* aux info at OFF always needs adjustment, no matter fast path -++ * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the -++ * original insn at old prog. -++ */ -++ old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); -++ -++ if (cnt == 1) -++ return 0; -++ prog_len = new_prog->len; -++ new_data = vzalloc(array_size(prog_len, -++ sizeof(struct bpf_insn_aux_data))); -++ if (!new_data) -++ return -ENOMEM; -++ memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); -++ memcpy(new_data + off + cnt - 1, old_data + off, -++ sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); -++ for (i = off; i < off + cnt - 1; i++) { -++ /* Expand insni[off]'s seen count to the patched range. */ -++ new_data[i].seen = old_seen; -++ new_data[i].zext_dst = insn_has_def32(env, insn + i); -++ } -++ env->insn_aux_data = new_data; -++ vfree(old_data); -++ return 0; -++} -++ -++static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) -++{ -++ int i; -++ -++ if (len == 1) -++ return; -++ /* NOTE: fake 'exit' subprog should be updated as well. */ -++ for (i = 0; i <= env->subprog_cnt; i++) { -++ if (env->subprog_info[i].start <= off) -+ continue; -++ env->subprog_info[i].start += len - 1; -++ } -++} -++ -++static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, -++ const struct bpf_insn *patch, u32 len) -++{ -++ struct bpf_prog *new_prog; -++ -++ new_prog = bpf_patch_insn_single(env->prog, off, patch, len); -++ if (IS_ERR(new_prog)) { -++ if (PTR_ERR(new_prog) == -ERANGE) -++ verbose(env, -++ "insn %d cannot be patched due to 16-bit range\n", -++ env->insn_aux_data[off].orig_idx); -++ return NULL; -++ } -++ if (adjust_insn_aux_data(env, new_prog, off, len)) -++ return NULL; -++ adjust_subprog_starts(env, off, len); -++ return new_prog; -++} -+ -+- /* adjust offset of jmps if necessary */ -+- if (i < pos && i + insn->off + 1 > pos) -+- insn->off += delta; -+- else if (i > pos + delta && i + insn->off + 1 <= pos + delta) -+- insn->off -= delta; -++static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, -++ u32 off, u32 cnt) -++{ -++ int i, j; -++ -++ /* find first prog starting at or after off (first to remove) */ -++ for (i = 0; i < env->subprog_cnt; i++) -++ if (env->subprog_info[i].start >= off) -++ break; -++ /* find first prog starting at or after off + cnt (first to stay) */ -++ for (j = i; j < env->subprog_cnt; j++) -++ if (env->subprog_info[j].start >= off + cnt) -++ break; -++ /* if j doesn't start exactly at off + cnt, we are just removing -++ * the front of previous prog -++ */ -++ if (env->subprog_info[j].start != off + cnt) -++ j--; -++ -++ if (j > i) { -++ struct bpf_prog_aux *aux = env->prog->aux; -++ int move; -++ -++ /* move fake 'exit' subprog as well */ -++ move = env->subprog_cnt + 1 - j; -++ -++ memmove(env->subprog_info + i, -++ env->subprog_info + j, -++ sizeof(*env->subprog_info) * move); -++ env->subprog_cnt -= j - i; -++ -++ /* remove func_info */ -++ if (aux->func_info) { -++ move = aux->func_info_cnt - j; -++ -++ memmove(aux->func_info + i, -++ aux->func_info + j, -++ sizeof(*aux->func_info) * move); -++ aux->func_info_cnt -= j - i; -++ /* func_info->insn_off is set after all code rewrites, -++ * in adjust_btf_func() - no need to adjust -++ */ -++ } -++ } else { -++ /* convert i from "first prog to remove" to "first to adjust" */ -++ if (env->subprog_info[i].start == off) -++ i++; -+ } -++ -++ /* update fake 'exit' subprog as well */ -++ for (; i <= env->subprog_cnt; i++) -++ env->subprog_info[i].start -= cnt; -++ -++ return 0; -++} -++ -++static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, -++ u32 cnt) -++{ -++ struct bpf_prog *prog = env->prog; -++ u32 i, l_off, l_cnt, nr_linfo; -++ struct bpf_line_info *linfo; -++ -++ nr_linfo = prog->aux->nr_linfo; -++ if (!nr_linfo) -++ return 0; -++ -++ linfo = prog->aux->linfo; -++ -++ /* find first line info to remove, count lines to be removed */ -++ for (i = 0; i < nr_linfo; i++) -++ if (linfo[i].insn_off >= off) -++ break; -++ -++ l_off = i; -++ l_cnt = 0; -++ for (; i < nr_linfo; i++) -++ if (linfo[i].insn_off < off + cnt) -++ l_cnt++; -++ else -++ break; -++ -++ /* First live insn doesn't match first live linfo, it needs to "inherit" -++ * last removed linfo. prog is already modified, so prog->len == off -++ * means no live instructions after (tail of the program was removed). -++ */ -++ if (prog->len != off && l_cnt && -++ (i == nr_linfo || linfo[i].insn_off != off + cnt)) { -++ l_cnt--; -++ linfo[--i].insn_off = off + cnt; -++ } -++ -++ /* remove the line info which refer to the removed instructions */ -++ if (l_cnt) { -++ memmove(linfo + l_off, linfo + i, -++ sizeof(*linfo) * (nr_linfo - i)); -++ -++ prog->aux->nr_linfo -= l_cnt; -++ nr_linfo = prog->aux->nr_linfo; -++ } -++ -++ /* pull all linfo[i].insn_off >= off + cnt in by cnt */ -++ for (i = l_off; i < nr_linfo; i++) -++ linfo[i].insn_off -= cnt; -++ -++ /* fix up all subprogs (incl. 'exit') which start >= off */ -++ for (i = 0; i <= env->subprog_cnt; i++) -++ if (env->subprog_info[i].linfo_idx > l_off) { -++ /* program may have started in the removed region but -++ * may not be fully removed -++ */ -++ if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) -++ env->subprog_info[i].linfo_idx -= l_cnt; -++ else -++ env->subprog_info[i].linfo_idx = l_off; -++ } -++ -++ return 0; -++} -++ -++static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) -++{ -++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; -++ unsigned int orig_prog_len = env->prog->len; -++ int err; -++ -++ err = bpf_remove_insns(env->prog, off, cnt); -++ if (err) -++ return err; -++ -++ err = adjust_subprog_starts_after_remove(env, off, cnt); -++ if (err) -++ return err; -++ -++ err = bpf_adj_linfo_after_remove(env, off, cnt); -++ if (err) -++ return err; -++ -++ memmove(aux_data + off, aux_data + off + cnt, -++ sizeof(*aux_data) * (orig_prog_len - off - cnt)); -++ -++ return 0; -+ } -+ -+-/* convert load instructions that access fields of 'struct __sk_buff' -+- * into sequence of instructions that access fields of 'struct sk_buff' -++/* The verifier does more data flow analysis than llvm and will not -++ * explore branches that are dead at run time. Malicious programs can -++ * have dead code too. Therefore replace all dead at-run-time code -++ * with 'ja -1'. -++ * -++ * Just nops are not optimal, e.g. if they would sit at the end of the -++ * program and through another bug we would manage to jump there, then -++ * we'd execute beyond program memory otherwise. Returning exception -++ * code also wouldn't work since we can have subprogs where the dead -++ * code could be located. -+ */ -+-static int convert_ctx_accesses(struct verifier_env *env) -++static void sanitize_dead_code(struct bpf_verifier_env *env) -+ { -++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; -++ struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); -+ struct bpf_insn *insn = env->prog->insnsi; -++ const int insn_cnt = env->prog->len; -++ int i; -++ -++ for (i = 0; i < insn_cnt; i++) { -++ if (aux_data[i].seen) -++ continue; -++ memcpy(insn + i, &trap, sizeof(trap)); -++ } -++} -++ -++static bool insn_is_cond_jump(u8 code) -++{ -++ u8 op; -++ -++ if (BPF_CLASS(code) == BPF_JMP32) -++ return true; -++ -++ if (BPF_CLASS(code) != BPF_JMP) -++ return false; -++ -++ op = BPF_OP(code); -++ return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; -++} -++ -++static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) -++{ -++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; -++ struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); -++ struct bpf_insn *insn = env->prog->insnsi; -++ const int insn_cnt = env->prog->len; -++ int i; -++ -++ for (i = 0; i < insn_cnt; i++, insn++) { -++ if (!insn_is_cond_jump(insn->code)) -++ continue; -++ -++ if (!aux_data[i + 1].seen) -++ ja.off = insn->off; -++ else if (!aux_data[i + 1 + insn->off].seen) -++ ja.off = 0; -++ else -++ continue; -++ -++ memcpy(insn, &ja, sizeof(ja)); -++ } -++} -++ -++static int opt_remove_dead_code(struct bpf_verifier_env *env) -++{ -++ struct bpf_insn_aux_data *aux_data = env->insn_aux_data; -+ int insn_cnt = env->prog->len; -+- struct bpf_insn insn_buf[16]; -++ int i, err; -++ -++ for (i = 0; i < insn_cnt; i++) { -++ int j; -++ -++ j = 0; -++ while (i + j < insn_cnt && !aux_data[i + j].seen) -++ j++; -++ if (!j) -++ continue; -++ -++ err = verifier_remove_insns(env, i, j); -++ if (err) -++ return err; -++ insn_cnt = env->prog->len; -++ } -++ -++ return 0; -++} -++ -++static int opt_remove_nops(struct bpf_verifier_env *env) -++{ -++ const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); -++ struct bpf_insn *insn = env->prog->insnsi; -++ int insn_cnt = env->prog->len; -++ int i, err; -++ -++ for (i = 0; i < insn_cnt; i++) { -++ if (memcmp(&insn[i], &ja, sizeof(ja))) -++ continue; -++ -++ err = verifier_remove_insns(env, i, 1); -++ if (err) -++ return err; -++ insn_cnt--; -++ i--; -++ } -++ -++ return 0; -++} -++ -++static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, -++ const union bpf_attr *attr) -++{ -++ struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; -++ struct bpf_insn_aux_data *aux = env->insn_aux_data; -++ int i, patch_len, delta = 0, len = env->prog->len; -++ struct bpf_insn *insns = env->prog->insnsi; -++ struct bpf_prog *new_prog; -++ bool rnd_hi32; -++ -++ rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; -++ zext_patch[1] = BPF_ZEXT_REG(0); -++ rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); -++ rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); -++ rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); -++ for (i = 0; i < len; i++) { -++ int adj_idx = i + delta; -++ struct bpf_insn insn; -++ -++ insn = insns[adj_idx]; -++ if (!aux[adj_idx].zext_dst) { -++ u8 code, class; -++ u32 imm_rnd; -++ -++ if (!rnd_hi32) -++ continue; -++ -++ code = insn.code; -++ class = BPF_CLASS(code); -++ if (insn_no_def(&insn)) -++ continue; -++ -++ /* NOTE: arg "reg" (the fourth one) is only used for -++ * BPF_STX which has been ruled out in above -++ * check, it is safe to pass NULL here. -++ */ -++ if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) { -++ if (class == BPF_LD && -++ BPF_MODE(code) == BPF_IMM) -++ i++; -++ continue; -++ } -++ -++ /* ctx load could be transformed into wider load. */ -++ if (class == BPF_LDX && -++ aux[adj_idx].ptr_type == PTR_TO_CTX) -++ continue; -++ -++ imm_rnd = get_random_int(); -++ rnd_hi32_patch[0] = insn; -++ rnd_hi32_patch[1].imm = imm_rnd; -++ rnd_hi32_patch[3].dst_reg = insn.dst_reg; -++ patch = rnd_hi32_patch; -++ patch_len = 4; -++ goto apply_patch_buffer; -++ } -++ -++ if (!bpf_jit_needs_zext()) -++ continue; -++ -++ zext_patch[0] = insn; -++ zext_patch[1].dst_reg = insn.dst_reg; -++ zext_patch[1].src_reg = insn.dst_reg; -++ patch = zext_patch; -++ patch_len = 2; -++apply_patch_buffer: -++ new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); -++ if (!new_prog) -++ return -ENOMEM; -++ env->prog = new_prog; -++ insns = new_prog->insnsi; -++ aux = env->insn_aux_data; -++ delta += patch_len - 1; -++ } -++ -++ return 0; -++} -++ -++/* convert load instructions that access fields of a context type into a -++ * sequence of instructions that access fields of the underlying structure: -++ * struct __sk_buff -> struct sk_buff -++ * struct bpf_sock_ops -> struct sock -++ */ -++static int convert_ctx_accesses(struct bpf_verifier_env *env) -++{ -++ const struct bpf_verifier_ops *ops = env->ops; -++ int i, cnt, size, ctx_field_size, delta = 0; -++ const int insn_cnt = env->prog->len; -++ struct bpf_insn insn_buf[16], *insn; -++ u32 target_size, size_default, off; -+ struct bpf_prog *new_prog; -+- u32 cnt; -+- int i; -+ enum bpf_access_type type; -++ bool is_narrower_load; -++ -++ if (ops->gen_prologue || env->seen_direct_write) { -++ if (!ops->gen_prologue) { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } -++ cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, -++ env->prog); -++ if (cnt >= ARRAY_SIZE(insn_buf)) { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } else if (cnt) { -++ new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ env->prog = new_prog; -++ delta += cnt - 1; -++ } -++ } -+ -+- if (!env->prog->aux->ops->convert_ctx_access) -++ if (bpf_prog_is_dev_bound(env->prog->aux)) -+ return 0; -+ -++ insn = env->prog->insnsi + delta; -++ -+ for (i = 0; i < insn_cnt; i++, insn++) { -+- if (insn->code == (BPF_LDX | BPF_MEM | BPF_W)) -++ bpf_convert_ctx_access_t convert_ctx_access; -++ -++ if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || -++ insn->code == (BPF_LDX | BPF_MEM | BPF_H) || -++ insn->code == (BPF_LDX | BPF_MEM | BPF_W) || -++ insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) -+ type = BPF_READ; -+- else if (insn->code == (BPF_STX | BPF_MEM | BPF_W)) -++ else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || -++ insn->code == (BPF_STX | BPF_MEM | BPF_H) || -++ insn->code == (BPF_STX | BPF_MEM | BPF_W) || -++ insn->code == (BPF_STX | BPF_MEM | BPF_DW)) -+ type = BPF_WRITE; -+ else -+ continue; -+ -+- if (insn->imm != PTR_TO_CTX) { -+- /* clear internal mark */ -+- insn->imm = 0; -++ if (type == BPF_WRITE && -++ env->insn_aux_data[i + delta].sanitize_stack_off) { -++ struct bpf_insn patch[] = { -++ /* Sanitize suspicious stack slot with zero. -++ * There are no memory dependencies for this store, -++ * since it's only using frame pointer and immediate -++ * constant of zero -++ */ -++ BPF_ST_MEM(BPF_DW, BPF_REG_FP, -++ env->insn_aux_data[i + delta].sanitize_stack_off, -++ 0), -++ /* the original STX instruction will immediately -++ * overwrite the same stack slot with appropriate value -++ */ -++ *insn, -++ }; -++ -++ cnt = ARRAY_SIZE(patch); -++ new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ switch (env->insn_aux_data[i + delta].ptr_type) { -++ case PTR_TO_CTX: -++ if (!ops->convert_ctx_access) -++ continue; -++ convert_ctx_access = ops->convert_ctx_access; -++ break; -++ default: -+ continue; -+ } -+ -+- cnt = env->prog->aux->ops-> -+- convert_ctx_access(type, insn->dst_reg, insn->src_reg, -+- insn->off, insn_buf, env->prog); -+- if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { -+- verbose("bpf verifier is misconfigured\n"); -++ ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; -++ size = BPF_LDST_BYTES(insn); -++ -++ /* If the read access is a narrower load of the field, -++ * convert to a 4/8-byte load, to minimum program type specific -++ * convert_ctx_access changes. If conversion is successful, -++ * we will apply proper mask to the result. -++ */ -++ is_narrower_load = size < ctx_field_size; -++ size_default = bpf_ctx_off_adjust_machine(ctx_field_size); -++ off = insn->off; -++ if (is_narrower_load) { -++ u8 size_code; -++ -++ if (type == BPF_WRITE) { -++ verbose(env, "bpf verifier narrow ctx access misconfigured\n"); -++ return -EINVAL; -++ } -++ -++ size_code = BPF_H; -++ if (ctx_field_size == 4) -++ size_code = BPF_W; -++ else if (ctx_field_size == 8) -++ size_code = BPF_DW; -++ -++ insn->off = off & ~(size_default - 1); -++ insn->code = BPF_LDX | BPF_MEM | size_code; -++ } -++ -++ target_size = 0; -++ cnt = convert_ctx_access(type, insn, insn_buf, env->prog, -++ &target_size); -++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || -++ (ctx_field_size && !target_size)) { -++ verbose(env, "bpf verifier is misconfigured\n"); -+ return -EINVAL; -+ } -+ -+- if (cnt == 1) { -+- memcpy(insn, insn_buf, sizeof(*insn)); -+- continue; -++ if (is_narrower_load && size < target_size) { -++ u8 shift = bpf_ctx_narrow_access_offset( -++ off, size, size_default) * 8; -++ if (ctx_field_size <= 4) { -++ if (shift) -++ insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, -++ insn->dst_reg, -++ shift); -++ insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, -++ (1 << size * 8) - 1); -++ } else { -++ if (shift) -++ insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, -++ insn->dst_reg, -++ shift); -++ insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, -++ (1ULL << size * 8) - 1); -++ } -+ } -+ -+- /* several new insns need to be inserted. Make room for them */ -+- insn_cnt += cnt - 1; -+- new_prog = bpf_prog_realloc(env->prog, -+- bpf_prog_size(insn_cnt), -+- GFP_USER); -++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); -+ if (!new_prog) -+ return -ENOMEM; -+ -+- new_prog->len = insn_cnt; -++ delta += cnt - 1; -+ -+- memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1, -+- sizeof(*insn) * (insn_cnt - i - cnt)); -++ /* keep walking new program and skip insns we just inserted */ -++ env->prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ } -+ -+- /* copy substitute insns in place of load instruction */ -+- memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt); -++ return 0; -++} -+ -+- /* adjust branches in the whole program */ -+- adjust_branches(new_prog, i, cnt - 1); -++static int jit_subprogs(struct bpf_verifier_env *env) -++{ -++ struct bpf_prog *prog = env->prog, **func, *tmp; -++ int i, j, subprog_start, subprog_end = 0, len, subprog; -++ struct bpf_insn *insn; -++ void *old_bpf_func; -++ int err; -+ -+- /* keep walking new program and skip insns we just inserted */ -+- env->prog = new_prog; -+- insn = new_prog->insnsi + i + cnt - 1; -+- i += cnt - 1; -++ if (env->subprog_cnt <= 1) -++ return 0; -++ -++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { -++ if (insn->code != (BPF_JMP | BPF_CALL) || -++ insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ /* Upon error here we cannot fall back to interpreter but -++ * need a hard reject of the program. Thus -EFAULT is -++ * propagated in any case. -++ */ -++ subprog = find_subprog(env, i + insn->imm + 1); -++ if (subprog < 0) { -++ WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", -++ i + insn->imm + 1); -++ return -EFAULT; -++ } -++ /* temporarily remember subprog id inside insn instead of -++ * aux_data, since next loop will split up all insns into funcs -++ */ -++ insn->off = subprog; -++ /* remember original imm in case JIT fails and fallback -++ * to interpreter will be needed -++ */ -++ env->insn_aux_data[i].call_imm = insn->imm; -++ /* point imm to __bpf_call_base+1 from JITs point of view */ -++ insn->imm = 1; -++ } -++ -++ err = bpf_prog_alloc_jited_linfo(prog); -++ if (err) -++ goto out_undo_insn; -++ -++ err = -ENOMEM; -++ func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); -++ if (!func) -++ goto out_undo_insn; -++ -++ for (i = 0; i < env->subprog_cnt; i++) { -++ subprog_start = subprog_end; -++ subprog_end = env->subprog_info[i + 1].start; -++ -++ len = subprog_end - subprog_start; -++ /* BPF_PROG_RUN doesn't call subprogs directly, -++ * hence main prog stats include the runtime of subprogs. -++ * subprogs don't have IDs and not reachable via prog_get_next_id -++ * func[i]->aux->stats will never be accessed and stays NULL -++ */ -++ func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); -++ if (!func[i]) -++ goto out_free; -++ memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], -++ len * sizeof(struct bpf_insn)); -++ func[i]->type = prog->type; -++ func[i]->len = len; -++ if (bpf_prog_calc_tag(func[i])) -++ goto out_free; -++ func[i]->is_func = 1; -++ func[i]->aux->func_idx = i; -++ /* the btf and func_info will be freed only at prog->aux */ -++ func[i]->aux->btf = prog->aux->btf; -++ func[i]->aux->func_info = prog->aux->func_info; -++ -++ /* Use bpf_prog_F_tag to indicate functions in stack traces. -++ * Long term would need debug info to populate names -++ */ -++ func[i]->aux->name[0] = 'F'; -++ func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; -++ func[i]->jit_requested = 1; -++ func[i]->aux->linfo = prog->aux->linfo; -++ func[i]->aux->nr_linfo = prog->aux->nr_linfo; -++ func[i]->aux->jited_linfo = prog->aux->jited_linfo; -++ func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; -++ func[i] = bpf_int_jit_compile(func[i]); -++ if (!func[i]->jited) { -++ err = -ENOTSUPP; -++ goto out_free; -++ } -++ cond_resched(); -++ } -++ /* at this point all bpf functions were successfully JITed -++ * now populate all bpf_calls with correct addresses and -++ * run last pass of JIT -++ */ -++ for (i = 0; i < env->subprog_cnt; i++) { -++ insn = func[i]->insnsi; -++ for (j = 0; j < func[i]->len; j++, insn++) { -++ if (insn->code != (BPF_JMP | BPF_CALL) || -++ insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ subprog = insn->off; -++ insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - -++ __bpf_call_base; -++ } -++ -++ /* we use the aux data to keep a list of the start addresses -++ * of the JITed images for each function in the program -++ * -++ * for some architectures, such as powerpc64, the imm field -++ * might not be large enough to hold the offset of the start -++ * address of the callee's JITed image from __bpf_call_base -++ * -++ * in such cases, we can lookup the start address of a callee -++ * by using its subprog id, available from the off field of -++ * the call instruction, as an index for this list -++ */ -++ func[i]->aux->func = func; -++ func[i]->aux->func_cnt = env->subprog_cnt; -++ } -++ for (i = 0; i < env->subprog_cnt; i++) { -++ old_bpf_func = func[i]->bpf_func; -++ tmp = bpf_int_jit_compile(func[i]); -++ if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { -++ verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); -++ err = -ENOTSUPP; -++ goto out_free; -++ } -++ cond_resched(); -++ } -++ -++ /* finally lock prog and jit images for all functions and -++ * populate kallsysm -++ */ -++ for (i = 0; i < env->subprog_cnt; i++) { -++ bpf_prog_lock_ro(func[i]); -++ bpf_prog_kallsyms_add(func[i]); -++ } -++ -++ /* Last step: make now unused interpreter insns from main -++ * prog consistent for later dump requests, so they can -++ * later look the same as if they were interpreted only. -++ */ -++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { -++ if (insn->code != (BPF_JMP | BPF_CALL) || -++ insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ insn->off = env->insn_aux_data[i].call_imm; -++ subprog = find_subprog(env, i + insn->off + 1); -++ insn->imm = subprog; -+ } -+ -++ prog->jited = 1; -++ prog->bpf_func = func[0]->bpf_func; -++ prog->aux->func = func; -++ prog->aux->func_cnt = env->subprog_cnt; -++ bpf_prog_free_unused_jited_linfo(prog); -+ return 0; -++out_free: -++ for (i = 0; i < env->subprog_cnt; i++) -++ if (func[i]) -++ bpf_jit_free(func[i]); -++ kfree(func); -++out_undo_insn: -++ /* cleanup main prog to be interpreted */ -++ prog->jit_requested = 0; -++ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { -++ if (insn->code != (BPF_JMP | BPF_CALL) || -++ insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ insn->off = 0; -++ insn->imm = env->insn_aux_data[i].call_imm; -++ } -++ bpf_prog_free_jited_linfo(prog); -++ return err; -+ } -+ -+-static void free_states(struct verifier_env *env) -++static int fixup_call_args(struct bpf_verifier_env *env) -+ { -+- struct verifier_state_list *sl, *sln; -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++ struct bpf_prog *prog = env->prog; -++ struct bpf_insn *insn = prog->insnsi; -++ int i, depth; -++#endif -++ int err = 0; -++ -++ if (env->prog->jit_requested && -++ !bpf_prog_is_dev_bound(env->prog->aux)) { -++ err = jit_subprogs(env); -++ if (err == 0) -++ return 0; -++ if (err == -EFAULT) -++ return err; -++ } -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++ for (i = 0; i < prog->len; i++, insn++) { -++ if (insn->code != (BPF_JMP | BPF_CALL) || -++ insn->src_reg != BPF_PSEUDO_CALL) -++ continue; -++ depth = get_callee_stack_depth(env, insn, i); -++ if (depth < 0) -++ return depth; -++ bpf_patch_call_args(insn, depth); -++ } -++ err = 0; -++#endif -++ return err; -++} -++ -++/* fixup insn->imm field of bpf_call instructions -++ * and inline eligible helpers as explicit sequence of BPF instructions -++ * -++ * this function is called after eBPF program passed verification -++ */ -++static int fixup_bpf_calls(struct bpf_verifier_env *env) -++{ -++ struct bpf_prog *prog = env->prog; -++ struct bpf_insn *insn = prog->insnsi; -++ const struct bpf_func_proto *fn; -++ const int insn_cnt = prog->len; -++ const struct bpf_map_ops *ops; -++ struct bpf_insn_aux_data *aux; -++ struct bpf_insn insn_buf[16]; -++ struct bpf_prog *new_prog; -++ struct bpf_map *map_ptr; -++ int i, cnt, delta = 0; -++ -++ for (i = 0; i < insn_cnt; i++, insn++) { -++ if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || -++ insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || -++ insn->code == (BPF_ALU | BPF_MOD | BPF_X) || -++ insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { -++ bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; -++ bool isdiv = BPF_OP(insn->code) == BPF_DIV; -++ struct bpf_insn *patchlet; -++ struct bpf_insn chk_and_div[] = { -++ /* [R,W]x div 0 -> 0 */ -++ BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | -++ BPF_JNE | BPF_K, insn->src_reg, -++ 0, 2, 0), -++ BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), -++ BPF_JMP_IMM(BPF_JA, 0, 0, 1), -++ *insn, -++ }; -++ struct bpf_insn chk_and_mod[] = { -++ /* [R,W]x mod 0 -> [R,W]x */ -++ BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | -++ BPF_JEQ | BPF_K, insn->src_reg, -++ 0, 1 + (is64 ? 0 : 1), 0), -++ *insn, -++ BPF_JMP_IMM(BPF_JA, 0, 0, 1), -++ BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), -++ }; -++ -++ patchlet = isdiv ? chk_and_div : chk_and_mod; -++ cnt = isdiv ? ARRAY_SIZE(chk_and_div) : -++ ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); -++ -++ new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ if (BPF_CLASS(insn->code) == BPF_LD && -++ (BPF_MODE(insn->code) == BPF_ABS || -++ BPF_MODE(insn->code) == BPF_IND)) { -++ cnt = env->ops->gen_ld_abs(insn, insn_buf); -++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } -++ -++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || -++ insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { -++ const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; -++ const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; -++ struct bpf_insn insn_buf[16]; -++ struct bpf_insn *patch = &insn_buf[0]; -++ bool issrc, isneg, isimm; -++ u32 off_reg; -++ -++ aux = &env->insn_aux_data[i + delta]; -++ if (!aux->alu_state || -++ aux->alu_state == BPF_ALU_NON_POINTER) -++ continue; -++ -++ isneg = aux->alu_state & BPF_ALU_NEG_VALUE; -++ issrc = (aux->alu_state & BPF_ALU_SANITIZE) == -++ BPF_ALU_SANITIZE_SRC; -++ isimm = aux->alu_state & BPF_ALU_IMMEDIATE; -++ -++ off_reg = issrc ? insn->src_reg : insn->dst_reg; -++ if (isimm) { -++ *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); -++ } else { -++ if (isneg) -++ *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); -++ *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); -++ *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); -++ *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); -++ *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); -++ *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); -++ *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg); -++ } -++ if (!issrc) -++ *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); -++ insn->src_reg = BPF_REG_AX; -++ if (isneg) -++ insn->code = insn->code == code_add ? -++ code_sub : code_add; -++ *patch++ = *insn; -++ if (issrc && isneg && !isimm) -++ *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); -++ cnt = patch - insn_buf; -++ -++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ if (insn->code != (BPF_JMP | BPF_CALL)) -++ continue; -++ if (insn->src_reg == BPF_PSEUDO_CALL) -++ continue; -++ -++ if (insn->imm == BPF_FUNC_get_route_realm) -++ prog->dst_needed = 1; -++ if (insn->imm == BPF_FUNC_get_prandom_u32) -++ bpf_user_rnd_init_once(); -++ if (insn->imm == BPF_FUNC_override_return) -++ prog->kprobe_override = 1; -++ if (insn->imm == BPF_FUNC_tail_call) { -++ /* If we tail call into other programs, we -++ * cannot make any assumptions since they can -++ * be replaced dynamically during runtime in -++ * the program array. -++ */ -++ prog->cb_access = 1; -++ env->prog->aux->stack_depth = MAX_BPF_STACK; -++ env->prog->aux->max_pkt_offset = MAX_PACKET_OFF; -++ -++ /* mark bpf_tail_call as different opcode to avoid -++ * conditional branch in the interpeter for every normal -++ * call and to prevent accidental JITing by JIT compiler -++ * that doesn't support bpf_tail_call yet -++ */ -++ insn->imm = 0; -++ insn->code = BPF_JMP | BPF_TAIL_CALL; -++ -++ aux = &env->insn_aux_data[i + delta]; -++ if (!bpf_map_ptr_unpriv(aux)) -++ continue; -++ -++ /* instead of changing every JIT dealing with tail_call -++ * emit two extra insns: -++ * if (index >= max_entries) goto out; -++ * index &= array->index_mask; -++ * to avoid out-of-bounds cpu speculation -++ */ -++ if (bpf_map_ptr_poisoned(aux)) { -++ verbose(env, "tail_call abusing map_ptr\n"); -++ return -EINVAL; -++ } -++ -++ map_ptr = BPF_MAP_PTR(aux->map_state); -++ insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, -++ map_ptr->max_entries, 2); -++ insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, -++ container_of(map_ptr, -++ struct bpf_array, -++ map)->index_mask); -++ insn_buf[2] = *insn; -++ cnt = 3; -++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup -++ * and other inlining handlers are currently limited to 64 bit -++ * only. -++ */ -++ if (prog->jit_requested && BITS_PER_LONG == 64 && -++ (insn->imm == BPF_FUNC_map_lookup_elem || -++ insn->imm == BPF_FUNC_map_update_elem || -++ insn->imm == BPF_FUNC_map_delete_elem || -++ insn->imm == BPF_FUNC_map_push_elem || -++ insn->imm == BPF_FUNC_map_pop_elem || -++ insn->imm == BPF_FUNC_map_peek_elem)) { -++ aux = &env->insn_aux_data[i + delta]; -++ if (bpf_map_ptr_poisoned(aux)) -++ goto patch_call_imm; -++ -++ map_ptr = BPF_MAP_PTR(aux->map_state); -++ ops = map_ptr->ops; -++ if (insn->imm == BPF_FUNC_map_lookup_elem && -++ ops->map_gen_lookup) { -++ cnt = ops->map_gen_lookup(map_ptr, insn_buf); -++ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { -++ verbose(env, "bpf verifier is misconfigured\n"); -++ return -EINVAL; -++ } -++ -++ new_prog = bpf_patch_insn_data(env, i + delta, -++ insn_buf, cnt); -++ if (!new_prog) -++ return -ENOMEM; -++ -++ delta += cnt - 1; -++ env->prog = prog = new_prog; -++ insn = new_prog->insnsi + i + delta; -++ continue; -++ } -++ -++ BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, -++ (void *(*)(struct bpf_map *map, void *key))NULL)); -++ BUILD_BUG_ON(!__same_type(ops->map_delete_elem, -++ (int (*)(struct bpf_map *map, void *key))NULL)); -++ BUILD_BUG_ON(!__same_type(ops->map_update_elem, -++ (int (*)(struct bpf_map *map, void *key, void *value, -++ u64 flags))NULL)); -++ BUILD_BUG_ON(!__same_type(ops->map_push_elem, -++ (int (*)(struct bpf_map *map, void *value, -++ u64 flags))NULL)); -++ BUILD_BUG_ON(!__same_type(ops->map_pop_elem, -++ (int (*)(struct bpf_map *map, void *value))NULL)); -++ BUILD_BUG_ON(!__same_type(ops->map_peek_elem, -++ (int (*)(struct bpf_map *map, void *value))NULL)); -++ -++ switch (insn->imm) { -++ case BPF_FUNC_map_lookup_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - -++ __bpf_call_base; -++ continue; -++ case BPF_FUNC_map_update_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_update_elem) - -++ __bpf_call_base; -++ continue; -++ case BPF_FUNC_map_delete_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - -++ __bpf_call_base; -++ continue; -++ case BPF_FUNC_map_push_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_push_elem) - -++ __bpf_call_base; -++ continue; -++ case BPF_FUNC_map_pop_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - -++ __bpf_call_base; -++ continue; -++ case BPF_FUNC_map_peek_elem: -++ insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - -++ __bpf_call_base; -++ continue; -++ } -++ -++ goto patch_call_imm; -++ } -++ -++patch_call_imm: -++ fn = env->ops->get_func_proto(insn->imm, env->prog); -++ /* all functions that have prototype and verifier allowed -++ * programs to call them, must be real in-kernel functions -++ */ -++ if (!fn->func) { -++ verbose(env, -++ "kernel subsystem misconfigured func %s#%d\n", -++ func_id_name(insn->imm), insn->imm); -++ return -EFAULT; -++ } -++ insn->imm = fn->func - __bpf_call_base; -++ } -++ -++ return 0; -++} -++ -++static void free_states(struct bpf_verifier_env *env) -++{ -++ struct bpf_verifier_state_list *sl, *sln; -+ int i; -+ -++ sl = env->free_list; -++ while (sl) { -++ sln = sl->next; -++ free_verifier_state(&sl->state, false); -++ kfree(sl); -++ sl = sln; -++ } -++ -+ if (!env->explored_states) -+ return; -+ -+- for (i = 0; i < env->prog->len; i++) { -++ for (i = 0; i < state_htab_size(env); i++) { -+ sl = env->explored_states[i]; -+ -+- if (sl) -+- while (sl != STATE_LIST_MARK) { -+- sln = sl->next; -+- kfree(sl); -+- sl = sln; -+- } -++ while (sl) { -++ sln = sl->next; -++ free_verifier_state(&sl->state, false); -++ kfree(sl); -++ sl = sln; -++ } -+ } -+ -+- kfree(env->explored_states); -++ kvfree(env->explored_states); -+ } -+ -+-int bpf_check(struct bpf_prog **prog, union bpf_attr *attr) -++static void print_verification_stats(struct bpf_verifier_env *env) -+ { -+- char __user *log_ubuf = NULL; -+- struct verifier_env *env; -+- int ret = -EINVAL; -++ int i; -+ -+- if ((*prog)->len <= 0 || (*prog)->len > BPF_MAXINSNS) -+- return -E2BIG; -++ if (env->log.level & BPF_LOG_STATS) { -++ verbose(env, "verification time %lld usec\n", -++ div_u64(env->verification_time, 1000)); -++ verbose(env, "stack depth "); -++ for (i = 0; i < env->subprog_cnt; i++) { -++ u32 depth = env->subprog_info[i].stack_depth; -++ -++ verbose(env, "%d", depth); -++ if (i + 1 < env->subprog_cnt) -++ verbose(env, "+"); -++ } -++ verbose(env, "\n"); -++ } -++ verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " -++ "total_states %d peak_states %d mark_read %d\n", -++ env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, -++ env->max_states_per_insn, env->total_states, -++ env->peak_states, env->longest_mark_read_walk); -++} -++ -++int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, -++ union bpf_attr __user *uattr) -++{ -++ u64 start_time = ktime_get_ns(); -++ struct bpf_verifier_env *env; -++ struct bpf_verifier_log *log; -++ int i, len, ret = -EINVAL; -++ bool is_priv; -+ -+- /* 'struct verifier_env' can be global, but since it's not small, -++ /* no program is valid */ -++ if (ARRAY_SIZE(bpf_verifier_ops) == 0) -++ return -EINVAL; -++ -++ /* 'struct bpf_verifier_env' can be global, but since it's not small, -+ * allocate/free it every time bpf_check() is called -+ */ -+- env = kzalloc(sizeof(struct verifier_env), GFP_KERNEL); -++ env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); -+ if (!env) -+ return -ENOMEM; -++ log = &env->log; -+ -++ len = (*prog)->len; -++ env->insn_aux_data = -++ vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); -++ ret = -ENOMEM; -++ if (!env->insn_aux_data) -++ goto err_free_env; -++ for (i = 0; i < len; i++) -++ env->insn_aux_data[i].orig_idx = i; -+ env->prog = *prog; -++ env->ops = bpf_verifier_ops[env->prog->type]; -++ is_priv = capable(CAP_SYS_ADMIN); -+ -+ /* grab the mutex to protect few globals used by verifier */ -+- mutex_lock(&bpf_verifier_lock); -++ if (!is_priv) -++ mutex_lock(&bpf_verifier_lock); -+ -+ if (attr->log_level || attr->log_buf || attr->log_size) { -+ /* user requested verbose verifier output -+ * and supplied buffer to store the verification trace -+ */ -+- log_level = attr->log_level; -+- log_ubuf = (char __user *) (unsigned long) attr->log_buf; -+- log_size = attr->log_size; -+- log_len = 0; -++ log->level = attr->log_level; -++ log->ubuf = (char __user *) (unsigned long) attr->log_buf; -++ log->len_total = attr->log_size; -+ -+ ret = -EINVAL; -+- /* log_* values have to be sane */ -+- if (log_size < 128 || log_size > UINT_MAX >> 8 || -+- log_level == 0 || log_ubuf == NULL) -+- goto free_env; -+- -+- ret = -ENOMEM; -+- log_buf = vmalloc(log_size); -+- if (!log_buf) -+- goto free_env; -+- } else { -+- log_level = 0; -+- } -++ /* log attributes have to be sane */ -++ if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || -++ !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) -++ goto err_unlock; -++ } -++ -++ env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); -++ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) -++ env->strict_alignment = true; -++ if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) -++ env->strict_alignment = false; -++ -++ env->allow_ptr_leaks = is_priv; -++ -++ if (is_priv) -++ env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; -+ -+ ret = replace_map_fd_with_map_ptr(env); -+ if (ret < 0) -+ goto skip_full_check; -+ -+- env->explored_states = kcalloc(env->prog->len, -+- sizeof(struct verifier_state_list *), -++ env->explored_states = kcalloc(state_htab_size(env), -++ sizeof(struct bpf_verifier_state_list *), -+ GFP_USER); -+ ret = -ENOMEM; -+ if (!env->explored_states) -+ goto skip_full_check; -+ -+- ret = check_cfg(env); -++ ret = check_subprogs(env); -++ if (ret < 0) -++ goto skip_full_check; -++ -++ ret = check_btf_info(env, attr, uattr); -+ if (ret < 0) -+ goto skip_full_check; -+ -+- env->allow_ptr_leaks = capable(CAP_SYS_ADMIN); -++ ret = check_cfg(env); -++ if (ret < 0) -++ goto skip_full_check; -+ -+ ret = do_check(env); -++ if (env->cur_state) { -++ free_verifier_state(env->cur_state, true); -++ env->cur_state = NULL; -++ } -+ -+ skip_full_check: -+- while (pop_stack(env, NULL) >= 0); -++ while (!pop_stack(env, NULL, NULL)); -+ free_states(env); -+ -+ if (ret == 0) -++ ret = check_max_stack_depth(env); -++ -++ /* instruction rewrites happen after this point */ -++ if (is_priv) { -++ if (ret == 0) -++ opt_hard_wire_dead_code_branches(env); -++ if (ret == 0) -++ ret = opt_remove_dead_code(env); -++ if (ret == 0) -++ ret = opt_remove_nops(env); -++ } else { -++ if (ret == 0) -++ sanitize_dead_code(env); -++ } -++ -++ if (ret == 0) -+ /* program is valid, convert *(u32*)(ctx + off) accesses */ -+ ret = convert_ctx_accesses(env); -+ -+- if (log_level && log_len >= log_size - 1) { -+- BUG_ON(log_len >= log_size); -+- /* verifier log exceeded user supplied buffer */ -+- ret = -ENOSPC; -+- /* fall through to return what was recorded */ -++ if (ret == 0) -++ ret = fixup_bpf_calls(env); -++ -++ /* do 32-bit optimization after insn patching has done so those patched -++ * insns could be handled correctly. -++ */ -++ if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { -++ ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); -++ env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret -++ : false; -+ } -+ -+- /* copy verifier log back to user space including trailing zero */ -+- if (log_level && copy_to_user(log_ubuf, log_buf, log_len + 1) != 0) { -++ if (ret == 0) -++ ret = fixup_call_args(env); -++ -++ env->verification_time = ktime_get_ns() - start_time; -++ print_verification_stats(env); -++ -++ if (log->level && bpf_verifier_log_full(log)) -++ ret = -ENOSPC; -++ if (log->level && !log->ubuf) { -+ ret = -EFAULT; -+- goto free_log_buf; -++ goto err_release_maps; -+ } -+ -+ if (ret == 0 && env->used_map_cnt) { -+@@ -2297,7 +9580,7 @@ skip_full_check: -+ -+ if (!env->prog->aux->used_maps) { -+ ret = -ENOMEM; -+- goto free_log_buf; -++ goto err_release_maps; -+ } -+ -+ memcpy(env->prog->aux->used_maps, env->used_maps, -+@@ -2310,17 +9593,21 @@ skip_full_check: -+ convert_pseudo_ld_imm64(env); -+ } -+ -+-free_log_buf: -+- if (log_level) -+- vfree(log_buf); -+-free_env: -++ if (ret == 0) -++ adjust_btf_func(env); -++ -++err_release_maps: -+ if (!env->prog->aux->used_maps) -+ /* if we didn't copy map pointers into bpf_prog_info, release -+- * them now. Otherwise free_bpf_prog_info() will release them. -++ * them now. Otherwise free_used_maps() will release them. -+ */ -+ release_maps(env); -+ *prog = env->prog; -++err_unlock: -++ if (!is_priv) -++ mutex_unlock(&bpf_verifier_lock); -++ vfree(env->insn_aux_data); -++err_free_env: -+ kfree(env); -+- mutex_unlock(&bpf_verifier_lock); -+ return ret; -+ } -+--- a/include/linux/filter.h -++++ b/include/linux/filter.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -+ /* -+ * Linux Socket Filter Data Structures -+ */ -+@@ -7,16 +8,22 @@ -+ #include -+ -+ #include -++#include -+ #include -+ #include -+ #include -+ #include -+ #include -+ #include -+-#include -++#include -++#include -++#include -++#include -++#include -+ -+-#include -++#include -+ -++#include -+ #include -+ #include -+ -+@@ -24,6 +31,11 @@ struct sk_buff; -+ struct sock; -+ struct seccomp_data; -+ struct bpf_prog_aux; -++struct xdp_rxq_info; -++struct xdp_buff; -++struct sock_reuseport; -++struct ctl_table; -++struct ctl_table_header; -+ -+ /* ArgX, context and stack frame pointer register positions. Note, -+ * Arg1, Arg2, Arg3, etc are used as argument mappings of function -+@@ -40,7 +52,26 @@ struct bpf_prog_aux; -+ /* Additional register mappings for converted user programs. */ -+ #define BPF_REG_A BPF_REG_0 -+ #define BPF_REG_X BPF_REG_7 -+-#define BPF_REG_TMP BPF_REG_8 -++#define BPF_REG_TMP BPF_REG_2 /* scratch reg */ -++#define BPF_REG_D BPF_REG_8 /* data, callee-saved */ -++#define BPF_REG_H BPF_REG_9 /* hlen, callee-saved */ -++ -++/* Kernel hidden auxiliary/helper register. */ -++#define BPF_REG_AX MAX_BPF_REG -++#define MAX_BPF_EXT_REG (MAX_BPF_REG + 1) -++#define MAX_BPF_JIT_REG MAX_BPF_EXT_REG -++ -++/* unused opcode to mark special call to bpf_tail_call() helper */ -++#define BPF_TAIL_CALL 0xf0 -++ -++/* unused opcode to mark call to interpreter with arguments */ -++#define BPF_CALL_ARGS 0xe0 -++ -++/* As per nm, we expose JITed images as text (code) section for -++ * kallsyms. That way, tools like perf can find it to match -++ * addresses. -++ */ -++#define BPF_SYM_ELF_TYPE 't' -+ -+ /* BPF program can access up to 512 bytes of stack space. */ -+ #define MAX_BPF_STACK 512 -+@@ -129,6 +160,20 @@ struct bpf_prog_aux; -+ .off = 0, \ -+ .imm = IMM }) -+ -++/* Special form of mov32, used for doing explicit zero extension on dst. */ -++#define BPF_ZEXT_REG(DST) \ -++ ((struct bpf_insn) { \ -++ .code = BPF_ALU | BPF_MOV | BPF_X, \ -++ .dst_reg = DST, \ -++ .src_reg = DST, \ -++ .off = 0, \ -++ .imm = 1 }) -++ -++static inline bool insn_is_zext(const struct bpf_insn *insn) -++{ -++ return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; -++} -++ -+ /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */ -+ #define BPF_LD_IMM64(DST, IMM) \ -+ BPF_LD_IMM64_RAW(DST, 0, IMM) -+@@ -249,8 +294,51 @@ struct bpf_prog_aux; -+ .off = OFF, \ -+ .imm = IMM }) -+ -++/* Like BPF_JMP_REG, but with 32-bit wide operands for comparison. */ -++ -++#define BPF_JMP32_REG(OP, DST, SRC, OFF) \ -++ ((struct bpf_insn) { \ -++ .code = BPF_JMP32 | BPF_OP(OP) | BPF_X, \ -++ .dst_reg = DST, \ -++ .src_reg = SRC, \ -++ .off = OFF, \ -++ .imm = 0 }) -++ -++/* Like BPF_JMP_IMM, but with 32-bit wide operands for comparison. */ -++ -++#define BPF_JMP32_IMM(OP, DST, IMM, OFF) \ -++ ((struct bpf_insn) { \ -++ .code = BPF_JMP32 | BPF_OP(OP) | BPF_K, \ -++ .dst_reg = DST, \ -++ .src_reg = 0, \ -++ .off = OFF, \ -++ .imm = IMM }) -++ -++/* Unconditional jumps, goto pc + off16 */ -++ -++#define BPF_JMP_A(OFF) \ -++ ((struct bpf_insn) { \ -++ .code = BPF_JMP | BPF_JA, \ -++ .dst_reg = 0, \ -++ .src_reg = 0, \ -++ .off = OFF, \ -++ .imm = 0 }) -++ -++/* Relative call */ -++ -++#define BPF_CALL_REL(TGT) \ -++ ((struct bpf_insn) { \ -++ .code = BPF_JMP | BPF_CALL, \ -++ .dst_reg = 0, \ -++ .src_reg = BPF_PSEUDO_CALL, \ -++ .off = 0, \ -++ .imm = TGT }) -++ -+ /* Function call */ -+ -++#define BPF_CAST_CALL(x) \ -++ ((u64 (*)(u64, u64, u64, u64, u64))(x)) -++ -+ #define BPF_EMIT_CALL(FUNC) \ -+ ((struct bpf_insn) { \ -+ .code = BPF_JMP | BPF_CALL, \ -+@@ -303,6 +391,112 @@ struct bpf_prog_aux; -+ bpf_size; \ -+ }) -+ -++#define bpf_size_to_bytes(bpf_size) \ -++({ \ -++ int bytes = -EINVAL; \ -++ \ -++ if (bpf_size == BPF_B) \ -++ bytes = sizeof(u8); \ -++ else if (bpf_size == BPF_H) \ -++ bytes = sizeof(u16); \ -++ else if (bpf_size == BPF_W) \ -++ bytes = sizeof(u32); \ -++ else if (bpf_size == BPF_DW) \ -++ bytes = sizeof(u64); \ -++ \ -++ bytes; \ -++}) -++ -++#define BPF_SIZEOF(type) \ -++ ({ \ -++ const int __size = bytes_to_bpf_size(sizeof(type)); \ -++ BUILD_BUG_ON(__size < 0); \ -++ __size; \ -++ }) -++ -++#define BPF_FIELD_SIZEOF(type, field) \ -++ ({ \ -++ const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \ -++ BUILD_BUG_ON(__size < 0); \ -++ __size; \ -++ }) -++ -++#define BPF_LDST_BYTES(insn) \ -++ ({ \ -++ const int __size = bpf_size_to_bytes(BPF_SIZE((insn)->code)); \ -++ WARN_ON(__size < 0); \ -++ __size; \ -++ }) -++ -++#define __BPF_MAP_0(m, v, ...) v -++#define __BPF_MAP_1(m, v, t, a, ...) m(t, a) -++#define __BPF_MAP_2(m, v, t, a, ...) m(t, a), __BPF_MAP_1(m, v, __VA_ARGS__) -++#define __BPF_MAP_3(m, v, t, a, ...) m(t, a), __BPF_MAP_2(m, v, __VA_ARGS__) -++#define __BPF_MAP_4(m, v, t, a, ...) m(t, a), __BPF_MAP_3(m, v, __VA_ARGS__) -++#define __BPF_MAP_5(m, v, t, a, ...) m(t, a), __BPF_MAP_4(m, v, __VA_ARGS__) -++ -++#define __BPF_REG_0(...) __BPF_PAD(5) -++#define __BPF_REG_1(...) __BPF_MAP(1, __VA_ARGS__), __BPF_PAD(4) -++#define __BPF_REG_2(...) __BPF_MAP(2, __VA_ARGS__), __BPF_PAD(3) -++#define __BPF_REG_3(...) __BPF_MAP(3, __VA_ARGS__), __BPF_PAD(2) -++#define __BPF_REG_4(...) __BPF_MAP(4, __VA_ARGS__), __BPF_PAD(1) -++#define __BPF_REG_5(...) __BPF_MAP(5, __VA_ARGS__) -++ -++#define __BPF_MAP(n, ...) __BPF_MAP_##n(__VA_ARGS__) -++#define __BPF_REG(n, ...) __BPF_REG_##n(__VA_ARGS__) -++ -++#define __BPF_CAST(t, a) \ -++ (__force t) \ -++ (__force \ -++ typeof(__builtin_choose_expr(sizeof(t) == sizeof(unsigned long), \ -++ (unsigned long)0, (t)0))) a -++#define __BPF_V void -++#define __BPF_N -++ -++#define __BPF_DECL_ARGS(t, a) t a -++#define __BPF_DECL_REGS(t, a) u64 a -++ -++#define __BPF_PAD(n) \ -++ __BPF_MAP(n, __BPF_DECL_ARGS, __BPF_N, u64, __ur_1, u64, __ur_2, \ -++ u64, __ur_3, u64, __ur_4, u64, __ur_5) -++ -++#define BPF_CALL_x(x, name, ...) \ -++ static __always_inline \ -++ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \ -++ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)); \ -++ u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__)) \ -++ { \ -++ return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\ -++ } \ -++ static __always_inline \ -++ u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)) -++ -++#define BPF_CALL_0(name, ...) BPF_CALL_x(0, name, __VA_ARGS__) -++#define BPF_CALL_1(name, ...) BPF_CALL_x(1, name, __VA_ARGS__) -++#define BPF_CALL_2(name, ...) BPF_CALL_x(2, name, __VA_ARGS__) -++#define BPF_CALL_3(name, ...) BPF_CALL_x(3, name, __VA_ARGS__) -++#define BPF_CALL_4(name, ...) BPF_CALL_x(4, name, __VA_ARGS__) -++#define BPF_CALL_5(name, ...) BPF_CALL_x(5, name, __VA_ARGS__) -++ -++#define bpf_ctx_range(TYPE, MEMBER) \ -++ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 -++#define bpf_ctx_range_till(TYPE, MEMBER1, MEMBER2) \ -++ offsetof(TYPE, MEMBER1) ... offsetofend(TYPE, MEMBER2) - 1 -++#if BITS_PER_LONG == 64 -++# define bpf_ctx_range_ptr(TYPE, MEMBER) \ -++ offsetof(TYPE, MEMBER) ... offsetofend(TYPE, MEMBER) - 1 -++#else -++# define bpf_ctx_range_ptr(TYPE, MEMBER) \ -++ offsetof(TYPE, MEMBER) ... offsetof(TYPE, MEMBER) + 8 - 1 -++#endif /* BITS_PER_LONG == 64 */ -++ -++#define bpf_target_off(TYPE, MEMBER, SIZE, PTR_SIZE) \ -++ ({ \ -++ BUILD_BUG_ON(FIELD_SIZEOF(TYPE, MEMBER) != (SIZE)); \ -++ *(PTR_SIZE) = (SIZE); \ -++ offsetof(TYPE, MEMBER); \ -++ }) -++ -+ #ifdef CONFIG_COMPAT -+ /* A struct sock_filter is architecture independent. */ -+ struct compat_sock_fprog { -+@@ -317,24 +511,33 @@ struct sock_fprog_kern { -+ }; -+ -+ struct bpf_binary_header { -+- unsigned int pages; -+- u8 image[]; -++ u32 pages; -++ /* Some arches need word alignment for their instructions */ -++ u8 image[] __aligned(4); -+ }; -+ -+ struct bpf_prog { -+ u16 pages; /* Number of allocated pages */ -+- kmemcheck_bitfield_begin(meta); -+ u16 jited:1, /* Is our filter JIT'ed? */ -++ jit_requested:1,/* archs need to JIT the prog */ -++ undo_set_mem:1, /* Passed set_memory_ro() checkpoint */ -+ gpl_compatible:1, /* Is filter GPL compatible? */ -+ cb_access:1, /* Is control block accessed? */ -+- dst_needed:1; /* Do we need dst entry? */ -+- kmemcheck_bitfield_end(meta); -+- u32 len; /* Number of filter blocks */ -++ dst_needed:1, /* Do we need dst entry? */ -++ blinded:1, /* Was blinded */ -++ is_func:1, /* program is a bpf function */ -++ kprobe_override:1, /* Do we override a kprobe? */ -++ has_callchain_buf:1, /* callchain buffer allocated? */ -++ enforce_expected_attach_type:1; /* Enforce expected_attach_type checking at attach time */ -+ enum bpf_prog_type type; /* Type of BPF program */ -++ enum bpf_attach_type expected_attach_type; /* For some prog types */ -++ u32 len; /* Number of filter blocks */ -++ u32 jited_len; /* Size of jited insns in bytes */ -++ u8 tag[BPF_TAG_SIZE]; -+ struct bpf_prog_aux *aux; /* Auxiliary fields */ -+ struct sock_fprog_kern *orig_prog; /* Original BPF program */ -+- unsigned int (*bpf_func)(const struct sk_buff *skb, -+- const struct bpf_insn *filter); -++ unsigned int (*bpf_func)(const void *ctx, -++ const struct bpf_insn *insn); -+ /* Instructions for interpreter */ -+ union { -+ struct sock_filter insns[0]; -+@@ -343,44 +546,160 @@ struct bpf_prog { -+ }; -+ -+ struct sk_filter { -+- atomic_t refcnt; -++ refcount_t refcnt; -+ struct rcu_head rcu; -+ struct bpf_prog *prog; -+ }; -+ -+-#define BPF_PROG_RUN(filter, ctx) (*filter->bpf_func)(ctx, filter->insnsi) -++#define BPF_PROG_RUN(prog, ctx) ({ \ -++ u32 ret; \ -++ ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \ -++ ret; }) -++ -++#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN -++ -++struct bpf_skb_data_end { -++ struct qdisc_skb_cb qdisc_cb; -++ void *data_meta; -++ void *data_end; -++}; -+ -+-static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog, -+- struct sk_buff *skb) -++struct bpf_redirect_info { -++ u32 flags; -++ u32 tgt_index; -++ void *tgt_value; -++ struct bpf_map *map; -++ struct bpf_map *map_to_flush; -++ u32 kern_flags; -++}; -++ -++DECLARE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); -++ -++/* flags for bpf_redirect_info kern_flags */ -++#define BPF_RI_F_RF_NO_DIRECT BIT(0) /* no napi_direct on return_frame */ -++ -++/* Compute the linear packet data range [data, data_end) which -++ * will be accessed by various program types (cls_bpf, act_bpf, -++ * lwt, ...). Subsystems allowing direct data access must (!) -++ * ensure that cb[] area can be written to when BPF program is -++ * invoked (otherwise cb[] save/restore is necessary). -++ */ -++static inline void bpf_compute_data_pointers(struct sk_buff *skb) -+ { -+- u8 *cb_data = qdisc_skb_cb(skb)->data; -+- u8 saved_cb[QDISC_CB_PRIV_LEN]; -+- u32 res; -++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; -++ -++ BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb)); -++ cb->data_meta = skb->data; -++ cb->data_end = skb->data + skb_headlen(skb); -++} -+ -++/* Similar to bpf_compute_data_pointers(), except that save orginal -++ * data in cb->data and cb->meta_data for restore. -++ */ -++static inline void bpf_compute_and_save_data_end( -++ struct sk_buff *skb, void **saved_data_end) -++{ -++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; -++ -++ *saved_data_end = cb->data_end; -++ cb->data_end = skb->data + skb_headlen(skb); -++} -++ -++/* Restore data saved by bpf_compute_data_pointers(). */ -++static inline void bpf_restore_data_end( -++ struct sk_buff *skb, void *saved_data_end) -++{ -++ struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb; -++ -++ cb->data_end = saved_data_end; -++} -++ -++static inline u8 *bpf_skb_cb(struct sk_buff *skb) -++{ -++ /* eBPF programs may read/write skb->cb[] area to transfer meta -++ * data between tail calls. Since this also needs to work with -++ * tc, that scratch memory is mapped to qdisc_skb_cb's data area. -++ * -++ * In some socket filter cases, the cb unfortunately needs to be -++ * saved/restored so that protocol specific skb->cb[] data won't -++ * be lost. In any case, due to unpriviledged eBPF programs -++ * attached to sockets, we need to clear the bpf_skb_cb() area -++ * to not leak previous contents to user space. -++ */ -++ BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN); -+ BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != -+- QDISC_CB_PRIV_LEN); -++ FIELD_SIZEOF(struct qdisc_skb_cb, data)); -++ -++ return qdisc_skb_cb(skb)->data; -++} -++ -++static inline u32 __bpf_prog_run_save_cb(const struct bpf_prog *prog, -++ struct sk_buff *skb) -++{ -++ u8 *cb_data = bpf_skb_cb(skb); -++ u8 cb_saved[BPF_SKB_CB_LEN]; -++ u32 res; -+ -+ if (unlikely(prog->cb_access)) { -+- memcpy(saved_cb, cb_data, sizeof(saved_cb)); -+- memset(cb_data, 0, sizeof(saved_cb)); -++ memcpy(cb_saved, cb_data, sizeof(cb_saved)); -++ memset(cb_data, 0, sizeof(cb_saved)); -+ } -+ -+ res = BPF_PROG_RUN(prog, skb); -+ -+ if (unlikely(prog->cb_access)) -+- memcpy(cb_data, saved_cb, sizeof(saved_cb)); -++ memcpy(cb_data, cb_saved, sizeof(cb_saved)); -++ -++ return res; -++} -++ -++static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog, -++ struct sk_buff *skb) -++{ -++ u32 res; -+ -++ preempt_disable(); -++ res = __bpf_prog_run_save_cb(prog, skb); -++ preempt_enable(); -+ return res; -+ } -+ -+ static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog, -+ struct sk_buff *skb) -+ { -+- u8 *cb_data = qdisc_skb_cb(skb)->data; -++ u8 *cb_data = bpf_skb_cb(skb); -++ u32 res; -+ -+ if (unlikely(prog->cb_access)) -+- memset(cb_data, 0, QDISC_CB_PRIV_LEN); -+- return BPF_PROG_RUN(prog, skb); -++ memset(cb_data, 0, BPF_SKB_CB_LEN); -++ -++ preempt_disable(); -++ res = BPF_PROG_RUN(prog, skb); -++ preempt_enable(); -++ return res; -++} -++ -++static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog, -++ struct xdp_buff *xdp) -++{ -++ /* Caller needs to hold rcu_read_lock() (!), otherwise program -++ * can be released while still running, or map elements could be -++ * freed early while still having concurrent users. XDP fastpath -++ * already takes rcu_read_lock() when fetching the program, so -++ * it's not necessary here anymore. -++ */ -++ return BPF_PROG_RUN(prog, xdp); -++} -++ -++static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog) -++{ -++ return prog->len * sizeof(struct bpf_insn); -++} -++ -++static inline u32 bpf_prog_tag_scratch_size(const struct bpf_prog *prog) -++{ -++ return round_up(bpf_prog_insn_size(prog) + -++ sizeof(__be64) + 1, SHA_MESSAGE_BYTES); -+ } -+ -+ static inline unsigned int bpf_prog_size(unsigned int proglen) -+@@ -399,27 +718,77 @@ static inline bool bpf_prog_was_classic( -+ return prog->type == BPF_PROG_TYPE_UNSPEC; -+ } -+ -++static inline u32 bpf_ctx_off_adjust_machine(u32 size) -++{ -++ const u32 size_machine = sizeof(unsigned long); -++ -++ if (size > size_machine && size % size_machine == 0) -++ size = size_machine; -++ -++ return size; -++} -++ -++static inline bool -++bpf_ctx_narrow_access_ok(u32 off, u32 size, u32 size_default) -++{ -++ return size <= size_default && (size & (size - 1)) == 0; -++} -++ -++static inline u8 -++bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default) -++{ -++ u8 access_off = off & (size_default - 1); -++ -++#ifdef __LITTLE_ENDIAN -++ return access_off; -++#else -++ return size_default - (access_off + size); -++#endif -++} -++ -++#define bpf_ctx_wide_access_ok(off, size, type, field) \ -++ (size == sizeof(__u64) && \ -++ off >= offsetof(type, field) && \ -++ off + sizeof(__u64) <= offsetofend(type, field) && \ -++ off % sizeof(__u64) == 0) -++ -+ #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0])) -+ -+-#ifdef CONFIG_DEBUG_SET_MODULE_RONX -+ static inline void bpf_prog_lock_ro(struct bpf_prog *fp) -+ { -+- set_memory_ro((unsigned long)fp, fp->pages); -++#ifndef CONFIG_BPF_JIT_ALWAYS_ON -++ if (!fp->jited) { -++ fp->undo_set_mem = 1; -++ set_memory_ro((unsigned long)fp, fp->pages); -++ } -++#endif -+ } -+ -+ static inline void bpf_prog_unlock_ro(struct bpf_prog *fp) -+ { -+- set_memory_rw((unsigned long)fp, fp->pages); -++ if (fp->undo_set_mem) -++ set_memory_rw((unsigned long)fp, fp->pages); -+ } -+-#else -+-static inline void bpf_prog_lock_ro(struct bpf_prog *fp) -++ -++static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr) -+ { -++ set_memory_ro((unsigned long)hdr, hdr->pages); -++ set_memory_x((unsigned long)hdr, hdr->pages); -+ } -+ -+-static inline void bpf_prog_unlock_ro(struct bpf_prog *fp) -++static inline void bpf_jit_binary_unlock_ro(struct bpf_binary_header *hdr) -++{ -++ set_memory_rw((unsigned long)hdr, hdr->pages); -++} -++ -++static inline struct bpf_binary_header * -++bpf_jit_binary_hdr(const struct bpf_prog *fp) -+ { -++ unsigned long real_start = (unsigned long)fp->bpf_func; -++ unsigned long addr = real_start & PAGE_MASK; -++ -++ return (void *)addr; -+ } -+-#endif /* CONFIG_DEBUG_SET_MODULE_RONX */ -+ -+ int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap); -+ static inline int sk_filter(struct sock *sk, struct sk_buff *skb) -+@@ -427,10 +796,20 @@ static inline int sk_filter(struct sock -+ return sk_filter_trim_cap(sk, skb, 1); -+ } -+ -+-int bpf_prog_select_runtime(struct bpf_prog *fp); -++struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); -+ void bpf_prog_free(struct bpf_prog *fp); -+ -++bool bpf_opcode_in_insntable(u8 code); -++ -++void bpf_prog_free_linfo(struct bpf_prog *prog); -++void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, -++ const u32 *insn_to_jit_off); -++int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog); -++void bpf_prog_free_jited_linfo(struct bpf_prog *prog); -++void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog); -++ -+ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags); -++struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags); -+ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, -+ gfp_t gfp_extra_flags); -+ void __bpf_prog_free(struct bpf_prog *fp); -+@@ -450,12 +829,11 @@ int bpf_prog_create_from_user(struct bpf -+ void bpf_prog_destroy(struct bpf_prog *fp); -+ -+ int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk); -+-int __sk_attach_filter(struct sock_fprog *fprog, struct sock *sk, -+- bool locked); -+ int sk_attach_bpf(u32 ufd, struct sock *sk); -++int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk); -++int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk); -++void sk_reuseport_prog_free(struct bpf_prog *prog); -+ int sk_detach_filter(struct sock *sk); -+-int __sk_detach_filter(struct sock *sk, bool locked); -+- -+ int sk_get_filter(struct sock *sk, struct sock_filter __user *filter, -+ unsigned int len); -+ -+@@ -463,10 +841,100 @@ bool sk_filter_charge(struct sock *sk, s -+ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); -+ -+ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); -+-void bpf_int_jit_compile(struct bpf_prog *fp); -+-bool bpf_helper_changes_skb_data(void *func); -++#define __bpf_call_base_args \ -++ ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ -++ (void *)__bpf_call_base) -++ -++struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); -++void bpf_jit_compile(struct bpf_prog *prog); -++bool bpf_jit_needs_zext(void); -++bool bpf_helper_changes_pkt_data(void *func); -++ -++static inline bool bpf_dump_raw_ok(const struct cred *cred) -++{ -++ /* Reconstruction of call-sites is dependent on kallsyms, -++ * thus make dump the same restriction. -++ */ -++ return true; -++} -++ -++struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, -++ const struct bpf_insn *patch, u32 len); -++int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); -++ -++void bpf_clear_redirect_map(struct bpf_map *map); -++ -++static inline bool xdp_return_frame_no_direct(void) -++{ -++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); -++ -++ return ri->kern_flags & BPF_RI_F_RF_NO_DIRECT; -++} -++ -++static inline void xdp_set_return_frame_no_direct(void) -++{ -++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); -++ -++ ri->kern_flags |= BPF_RI_F_RF_NO_DIRECT; -++} -++ -++static inline void xdp_clear_return_frame_no_direct(void) -++{ -++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); -++ -++ ri->kern_flags &= ~BPF_RI_F_RF_NO_DIRECT; -++} -++ -++static inline int xdp_ok_fwd_dev(const struct net_device *fwd, -++ unsigned int pktlen) -++{ -++ unsigned int len; -++ -++ if (unlikely(!(fwd->flags & IFF_UP))) -++ return -ENETDOWN; -++ -++ len = fwd->mtu + fwd->hard_header_len + VLAN_HLEN; -++ if (pktlen > len) -++ return -EMSGSIZE; -++ -++ return 0; -++} -++ -++/* The pair of xdp_do_redirect and xdp_do_flush_map MUST be called in the -++ * same cpu context. Further for best results no more than a single map -++ * for the do_redirect/do_flush pair should be used. This limitation is -++ * because we only track one map and force a flush when the map changes. -++ * This does not appear to be a real limitation for existing software. -++ */ -++int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb, -++ struct xdp_buff *xdp, struct bpf_prog *prog); -++int xdp_do_redirect(struct net_device *dev, -++ struct xdp_buff *xdp, -++ struct bpf_prog *prog); -++void xdp_do_flush_map(void); -++ -++void bpf_warn_invalid_xdp_action(u32 act); -++ -++#ifdef CONFIG_INET -++struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, -++ struct bpf_prog *prog, struct sk_buff *skb, -++ u32 hash); -++#else -++static inline struct sock * -++bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk, -++ struct bpf_prog *prog, struct sk_buff *skb, -++ u32 hash) -++{ -++ return NULL; -++} -++#endif -+ -+ #ifdef CONFIG_BPF_JIT -++extern int bpf_jit_enable; -++extern int bpf_jit_harden; -++extern int bpf_jit_kallsyms; -++extern long bpf_jit_limit; -++ -+ typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); -+ -+ struct bpf_binary_header * -+@@ -474,10 +942,18 @@ bpf_jit_binary_alloc(unsigned int progle -+ unsigned int alignment, -+ bpf_jit_fill_hole_t bpf_fill_ill_insns); -+ void bpf_jit_binary_free(struct bpf_binary_header *hdr); -+- -+-void bpf_jit_compile(struct bpf_prog *fp); -++u64 bpf_jit_alloc_exec_limit(void); -++void *bpf_jit_alloc_exec(unsigned long size); -++void bpf_jit_free_exec(void *addr); -+ void bpf_jit_free(struct bpf_prog *fp); -+ -++int bpf_jit_get_func_addr(const struct bpf_prog *prog, -++ const struct bpf_insn *insn, bool extra_pass, -++ u64 *func_addr, bool *func_addr_fixed); -++ -++struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp); -++void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); -++ -+ static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, -+ u32 pass, void *image) -+ { -+@@ -488,17 +964,144 @@ static inline void bpf_jit_dump(unsigned -+ print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET, -+ 16, 1, image, proglen, false); -+ } -+-#else -+-static inline void bpf_jit_compile(struct bpf_prog *fp) -++ -++static inline bool bpf_jit_is_ebpf(void) -++{ -++# ifdef CONFIG_HAVE_EBPF_JIT -++ return true; -++# else -++ return false; -++# endif -++} -++ -++static inline bool ebpf_jit_enabled(void) -++{ -++ return bpf_jit_enable && bpf_jit_is_ebpf(); -++} -++ -++static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) -++{ -++ return fp->jited && bpf_jit_is_ebpf(); -++} -++ -++static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) -++{ -++ /* These are the prerequisites, should someone ever have the -++ * idea to call blinding outside of them, we make sure to -++ * bail out. -++ */ -++ if (!bpf_jit_is_ebpf()) -++ return false; -++ if (!prog->jit_requested) -++ return false; -++ if (!bpf_jit_harden) -++ return false; -++ if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN)) -++ return false; -++ -++ return true; -++} -++ -++static inline bool bpf_jit_kallsyms_enabled(void) -+ { -++ /* There are a couple of corner cases where kallsyms should -++ * not be enabled f.e. on hardening. -++ */ -++ if (bpf_jit_harden) -++ return false; -++ if (!bpf_jit_kallsyms) -++ return false; -++ if (bpf_jit_kallsyms == 1) -++ return true; -++ -++ return false; -++} -++ -++const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, -++ unsigned long *off, char *sym); -++bool is_bpf_text_address(unsigned long addr); -++int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, -++ char *sym); -++ -++static inline const char * -++bpf_address_lookup(unsigned long addr, unsigned long *size, -++ unsigned long *off, char **modname, char *sym) -++{ -++ const char *ret = __bpf_address_lookup(addr, size, off, sym); -++ -++ if (ret && modname) -++ *modname = NULL; -++ return ret; -++} -++ -++void bpf_prog_kallsyms_add(struct bpf_prog *fp); -++void bpf_prog_kallsyms_del(struct bpf_prog *fp); -++void bpf_get_prog_name(const struct bpf_prog *prog, char *sym); -++ -++#else /* CONFIG_BPF_JIT */ -++ -++static inline bool ebpf_jit_enabled(void) -++{ -++ return false; -++} -++ -++static inline bool bpf_prog_ebpf_jited(const struct bpf_prog *fp) -++{ -++ return false; -+ } -+ -+ static inline void bpf_jit_free(struct bpf_prog *fp) -+ { -+ bpf_prog_unlock_free(fp); -+ } -++ -++static inline bool bpf_jit_kallsyms_enabled(void) -++{ -++ return false; -++} -++ -++static inline const char * -++__bpf_address_lookup(unsigned long addr, unsigned long *size, -++ unsigned long *off, char *sym) -++{ -++ return NULL; -++} -++ -++static inline bool is_bpf_text_address(unsigned long addr) -++{ -++ return false; -++} -++ -++static inline int bpf_get_kallsym(unsigned int symnum, unsigned long *value, -++ char *type, char *sym) -++{ -++ return -ERANGE; -++} -++ -++static inline const char * -++bpf_address_lookup(unsigned long addr, unsigned long *size, -++ unsigned long *off, char **modname, char *sym) -++{ -++ return NULL; -++} -++ -++static inline void bpf_prog_kallsyms_add(struct bpf_prog *fp) -++{ -++} -++ -++static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) -++{ -++} -++ -++static inline void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) -++{ -++ sym[0] = '\0'; -++} -++ -+ #endif /* CONFIG_BPF_JIT */ -+ -++void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); -++ -+ #define BPF_ANC BIT(15) -+ -+ static inline bool bpf_needs_clear_a(const struct sock_filter *first) -+@@ -571,4 +1174,59 @@ static inline int bpf_tell_extensions(vo -+ return SKF_AD_MAX; -+ } -+ -++struct bpf_sock_addr_kern { -++ struct sock *sk; -++ struct sockaddr *uaddr; -++ /* Temporary "register" to make indirect stores to nested structures -++ * defined above. We need three registers to make such a store, but -++ * only two (src and dst) are available at convert_ctx_access time -++ */ -++ u64 tmp_reg; -++ void *t_ctx; /* Attach type specific context. */ -++}; -++ -++struct bpf_sock_ops_kern { -++ struct sock *sk; -++ u32 op; -++ union { -++ u32 args[4]; -++ u32 reply; -++ u32 replylong[4]; -++ }; -++ u32 is_fullsock; -++ u64 temp; /* temp and everything after is not -++ * initialized to 0 before calling -++ * the BPF program. New fields that -++ * should be initialized to 0 should -++ * be inserted before temp. -++ * temp is scratch storage used by -++ * sock_ops_convert_ctx_access -++ * as temporary storage of a register. -++ */ -++}; -++ -++struct bpf_sysctl_kern { -++ struct ctl_table_header *head; -++ struct ctl_table *table; -++ void *cur_val; -++ size_t cur_len; -++ void *new_val; -++ size_t new_len; -++ int new_updated; -++ int write; -++ loff_t *ppos; -++ /* Temporary "register" for indirect stores to ppos. */ -++ u64 tmp_reg; -++}; -++ -++struct bpf_sockopt_kern { -++ struct sock *sk; -++ u8 *optval; -++ u8 *optval_end; -++ s32 level; -++ s32 optname; -++ s32 optlen; -++ s32 retval; -++}; -++ -+ #endif /* __LINUX_FILTER_H__ */ -+--- /dev/null -++++ b/include/linux/set_memory.h -+@@ -0,0 +1,47 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* -++ * Copyright 2017, Michael Ellerman, IBM Corporation. -++ */ -++#ifndef _LINUX_SET_MEMORY_H_ -++#define _LINUX_SET_MEMORY_H_ -++ -++#include -++ -++#ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -++static inline int set_direct_map_invalid_noflush(struct page *page) -++{ -++ return 0; -++} -++static inline int set_direct_map_default_noflush(struct page *page) -++{ -++ return 0; -++} -++#endif -++ -++#ifndef set_mce_nospec -++static inline int set_mce_nospec(unsigned long pfn, bool unmap) -++{ -++ return 0; -++} -++#endif -++ -++#ifndef clear_mce_nospec -++static inline int clear_mce_nospec(unsigned long pfn) -++{ -++ return 0; -++} -++#endif -++ -++#ifndef CONFIG_ARCH_HAS_MEM_ENCRYPT -++static inline int set_memory_encrypted(unsigned long addr, int numpages) -++{ -++ return 0; -++} -++ -++static inline int set_memory_decrypted(unsigned long addr, int numpages) -++{ -++ return 0; -++} -++#endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */ -++ -++#endif /* _LINUX_SET_MEMORY_H_ */ -+--- /dev/null -++++ b/include/trace/events/xdp.h -+@@ -0,0 +1,407 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++#undef TRACE_SYSTEM -++#define TRACE_SYSTEM xdp -++ -++#if !defined(_TRACE_XDP_H) || defined(TRACE_HEADER_MULTI_READ) -++#define _TRACE_XDP_H -++ -++#include -++#include -++#include -++#include -++ -++#define __XDP_ACT_MAP(FN) \ -++ FN(ABORTED) \ -++ FN(DROP) \ -++ FN(PASS) \ -++ FN(TX) \ -++ FN(REDIRECT) -++ -++#define __XDP_ACT_TP_FN(x) \ -++ TRACE_DEFINE_ENUM(XDP_##x); -++#define __XDP_ACT_SYM_FN(x) \ -++ { XDP_##x, #x }, -++#define __XDP_ACT_SYM_TAB \ -++ __XDP_ACT_MAP(__XDP_ACT_SYM_FN) { -1, 0 } -++__XDP_ACT_MAP(__XDP_ACT_TP_FN) -++ -++TRACE_EVENT(xdp_exception, -++ -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, u32 act), -++ -++ TP_ARGS(dev, xdp, act), -++ -++ TP_STRUCT__entry( -++ __field(int, prog_id) -++ __field(u32, act) -++ __field(int, ifindex) -++ ), -++ -++ TP_fast_assign( -++ __entry->prog_id = xdp->aux->id; -++ __entry->act = act; -++ __entry->ifindex = dev->ifindex; -++ ), -++ -++ TP_printk("prog_id=%d action=%s ifindex=%d", -++ __entry->prog_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->ifindex) -++); -++ -++TRACE_EVENT(xdp_bulk_tx, -++ -++ TP_PROTO(const struct net_device *dev, -++ int sent, int drops, int err), -++ -++ TP_ARGS(dev, sent, drops, err), -++ -++ TP_STRUCT__entry( -++ __field(int, ifindex) -++ __field(u32, act) -++ __field(int, drops) -++ __field(int, sent) -++ __field(int, err) -++ ), -++ -++ TP_fast_assign( -++ __entry->ifindex = dev->ifindex; -++ __entry->act = XDP_TX; -++ __entry->drops = drops; -++ __entry->sent = sent; -++ __entry->err = err; -++ ), -++ -++ TP_printk("ifindex=%d action=%s sent=%d drops=%d err=%d", -++ __entry->ifindex, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->sent, __entry->drops, __entry->err) -++); -++ -++DECLARE_EVENT_CLASS(xdp_redirect_template, -++ -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, -++ int to_ifindex, int err, -++ const struct bpf_map *map, u32 map_index), -++ -++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), -++ -++ TP_STRUCT__entry( -++ __field(int, prog_id) -++ __field(u32, act) -++ __field(int, ifindex) -++ __field(int, err) -++ __field(int, to_ifindex) -++ __field(u32, map_id) -++ __field(int, map_index) -++ ), -++ -++ TP_fast_assign( -++ __entry->prog_id = xdp->aux->id; -++ __entry->act = XDP_REDIRECT; -++ __entry->ifindex = dev->ifindex; -++ __entry->err = err; -++ __entry->to_ifindex = to_ifindex; -++ __entry->map_id = map ? map->id : 0; -++ __entry->map_index = map_index; -++ ), -++ -++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d", -++ __entry->prog_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->ifindex, __entry->to_ifindex, -++ __entry->err) -++); -++ -++DEFINE_EVENT(xdp_redirect_template, xdp_redirect, -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, -++ int to_ifindex, int err, -++ const struct bpf_map *map, u32 map_index), -++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index) -++); -++ -++DEFINE_EVENT(xdp_redirect_template, xdp_redirect_err, -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, -++ int to_ifindex, int err, -++ const struct bpf_map *map, u32 map_index), -++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index) -++); -++ -++#define _trace_xdp_redirect(dev, xdp, to) \ -++ trace_xdp_redirect(dev, xdp, to, 0, NULL, 0); -++ -++#define _trace_xdp_redirect_err(dev, xdp, to, err) \ -++ trace_xdp_redirect_err(dev, xdp, to, err, NULL, 0); -++ -++DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map, -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, -++ int to_ifindex, int err, -++ const struct bpf_map *map, u32 map_index), -++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), -++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d" -++ " map_id=%d map_index=%d", -++ __entry->prog_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->ifindex, __entry->to_ifindex, -++ __entry->err, -++ __entry->map_id, __entry->map_index) -++); -++ -++DEFINE_EVENT_PRINT(xdp_redirect_template, xdp_redirect_map_err, -++ TP_PROTO(const struct net_device *dev, -++ const struct bpf_prog *xdp, -++ int to_ifindex, int err, -++ const struct bpf_map *map, u32 map_index), -++ TP_ARGS(dev, xdp, to_ifindex, err, map, map_index), -++ TP_printk("prog_id=%d action=%s ifindex=%d to_ifindex=%d err=%d" -++ " map_id=%d map_index=%d", -++ __entry->prog_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->ifindex, __entry->to_ifindex, -++ __entry->err, -++ __entry->map_id, __entry->map_index) -++); -++ -++#ifndef __DEVMAP_OBJ_TYPE -++#define __DEVMAP_OBJ_TYPE -++struct _bpf_dtab_netdev { -++ struct net_device *dev; -++}; -++#endif /* __DEVMAP_OBJ_TYPE */ -++ -++#define devmap_ifindex(fwd, map) \ -++ ((map->map_type == BPF_MAP_TYPE_DEVMAP || \ -++ map->map_type == BPF_MAP_TYPE_DEVMAP_HASH) ? \ -++ ((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0) -++ -++#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \ -++ trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \ -++ 0, map, idx) -++ -++#define _trace_xdp_redirect_map_err(dev, xdp, fwd, map, idx, err) \ -++ trace_xdp_redirect_map_err(dev, xdp, devmap_ifindex(fwd, map), \ -++ err, map, idx) -++ -++TRACE_EVENT(xdp_cpumap_kthread, -++ -++ TP_PROTO(int map_id, unsigned int processed, unsigned int drops, -++ int sched), -++ -++ TP_ARGS(map_id, processed, drops, sched), -++ -++ TP_STRUCT__entry( -++ __field(int, map_id) -++ __field(u32, act) -++ __field(int, cpu) -++ __field(unsigned int, drops) -++ __field(unsigned int, processed) -++ __field(int, sched) -++ ), -++ -++ TP_fast_assign( -++ __entry->map_id = map_id; -++ __entry->act = XDP_REDIRECT; -++ __entry->cpu = smp_processor_id(); -++ __entry->drops = drops; -++ __entry->processed = processed; -++ __entry->sched = sched; -++ ), -++ -++ TP_printk("kthread" -++ " cpu=%d map_id=%d action=%s" -++ " processed=%u drops=%u" -++ " sched=%d", -++ __entry->cpu, __entry->map_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->processed, __entry->drops, -++ __entry->sched) -++); -++ -++TRACE_EVENT(xdp_cpumap_enqueue, -++ -++ TP_PROTO(int map_id, unsigned int processed, unsigned int drops, -++ int to_cpu), -++ -++ TP_ARGS(map_id, processed, drops, to_cpu), -++ -++ TP_STRUCT__entry( -++ __field(int, map_id) -++ __field(u32, act) -++ __field(int, cpu) -++ __field(unsigned int, drops) -++ __field(unsigned int, processed) -++ __field(int, to_cpu) -++ ), -++ -++ TP_fast_assign( -++ __entry->map_id = map_id; -++ __entry->act = XDP_REDIRECT; -++ __entry->cpu = smp_processor_id(); -++ __entry->drops = drops; -++ __entry->processed = processed; -++ __entry->to_cpu = to_cpu; -++ ), -++ -++ TP_printk("enqueue" -++ " cpu=%d map_id=%d action=%s" -++ " processed=%u drops=%u" -++ " to_cpu=%d", -++ __entry->cpu, __entry->map_id, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->processed, __entry->drops, -++ __entry->to_cpu) -++); -++ -++TRACE_EVENT(xdp_devmap_xmit, -++ -++ TP_PROTO(const struct bpf_map *map, u32 map_index, -++ int sent, int drops, -++ const struct net_device *from_dev, -++ const struct net_device *to_dev, int err), -++ -++ TP_ARGS(map, map_index, sent, drops, from_dev, to_dev, err), -++ -++ TP_STRUCT__entry( -++ __field(int, map_id) -++ __field(u32, act) -++ __field(u32, map_index) -++ __field(int, drops) -++ __field(int, sent) -++ __field(int, from_ifindex) -++ __field(int, to_ifindex) -++ __field(int, err) -++ ), -++ -++ TP_fast_assign( -++ __entry->map_id = map->id; -++ __entry->act = XDP_REDIRECT; -++ __entry->map_index = map_index; -++ __entry->drops = drops; -++ __entry->sent = sent; -++ __entry->from_ifindex = from_dev->ifindex; -++ __entry->to_ifindex = to_dev->ifindex; -++ __entry->err = err; -++ ), -++ -++ TP_printk("ndo_xdp_xmit" -++ " map_id=%d map_index=%d action=%s" -++ " sent=%d drops=%d" -++ " from_ifindex=%d to_ifindex=%d err=%d", -++ __entry->map_id, __entry->map_index, -++ __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB), -++ __entry->sent, __entry->drops, -++ __entry->from_ifindex, __entry->to_ifindex, __entry->err) -++); -++ -++/* Expect users already include , but not xdp_priv.h */ -++#include -++ -++#define __MEM_TYPE_MAP(FN) \ -++ FN(PAGE_SHARED) \ -++ FN(PAGE_ORDER0) \ -++ FN(PAGE_POOL) \ -++ FN(ZERO_COPY) -++ -++#define __MEM_TYPE_TP_FN(x) \ -++ TRACE_DEFINE_ENUM(MEM_TYPE_##x); -++#define __MEM_TYPE_SYM_FN(x) \ -++ { MEM_TYPE_##x, #x }, -++#define __MEM_TYPE_SYM_TAB \ -++ __MEM_TYPE_MAP(__MEM_TYPE_SYM_FN) { -1, 0 } -++__MEM_TYPE_MAP(__MEM_TYPE_TP_FN) -++ -++TRACE_EVENT(mem_disconnect, -++ -++ TP_PROTO(const struct xdp_mem_allocator *xa), -++ -++ TP_ARGS(xa), -++ -++ TP_STRUCT__entry( -++ __field(const struct xdp_mem_allocator *, xa) -++ __field(u32, mem_id) -++ __field(u32, mem_type) -++ __field(const void *, allocator) -++ ), -++ -++ TP_fast_assign( -++ __entry->xa = xa; -++ __entry->mem_id = xa->mem.id; -++ __entry->mem_type = xa->mem.type; -++ __entry->allocator = xa->allocator; -++ ), -++ -++ TP_printk("mem_id=%d mem_type=%s allocator=%p", -++ __entry->mem_id, -++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), -++ __entry->allocator -++ ) -++); -++ -++TRACE_EVENT(mem_connect, -++ -++ TP_PROTO(const struct xdp_mem_allocator *xa, -++ const struct xdp_rxq_info *rxq), -++ -++ TP_ARGS(xa, rxq), -++ -++ TP_STRUCT__entry( -++ __field(const struct xdp_mem_allocator *, xa) -++ __field(u32, mem_id) -++ __field(u32, mem_type) -++ __field(const void *, allocator) -++ __field(const struct xdp_rxq_info *, rxq) -++ __field(int, ifindex) -++ ), -++ -++ TP_fast_assign( -++ __entry->xa = xa; -++ __entry->mem_id = xa->mem.id; -++ __entry->mem_type = xa->mem.type; -++ __entry->allocator = xa->allocator; -++ __entry->rxq = rxq; -++ __entry->ifindex = rxq->dev->ifindex; -++ ), -++ -++ TP_printk("mem_id=%d mem_type=%s allocator=%p" -++ " ifindex=%d", -++ __entry->mem_id, -++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), -++ __entry->allocator, -++ __entry->ifindex -++ ) -++); -++ -++TRACE_EVENT(mem_return_failed, -++ -++ TP_PROTO(const struct xdp_mem_info *mem, -++ const struct page *page), -++ -++ TP_ARGS(mem, page), -++ -++ TP_STRUCT__entry( -++ __field(const struct page *, page) -++ __field(u32, mem_id) -++ __field(u32, mem_type) -++ ), -++ -++ TP_fast_assign( -++ __entry->page = page; -++ __entry->mem_id = mem->id; -++ __entry->mem_type = mem->type; -++ ), -++ -++ TP_printk("mem_id=%d mem_type=%s page=%p", -++ __entry->mem_id, -++ __print_symbolic(__entry->mem_type, __MEM_TYPE_SYM_TAB), -++ __entry->page -++ ) -++); -++ -++#endif /* _TRACE_XDP_H */ -++ -++#include -+--- /dev/null -++++ b/include/net/xdp_priv.h -+@@ -0,0 +1,20 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -++#ifndef __LINUX_NET_XDP_PRIV_H__ -++#define __LINUX_NET_XDP_PRIV_H__ -++ -++#include -++#include -++ -++/* Private to net/core/xdp.c, but used by trace/events/xdp.h */ -++struct xdp_mem_allocator { -++ struct xdp_mem_info mem; -++ union { -++ void *allocator; -++ struct page_pool *page_pool; -++ struct zero_copy_allocator *zc_alloc; -++ }; -++ struct rhash_head node; -++ struct rcu_head rcu; -++}; -++ -++#endif /* __LINUX_NET_XDP_PRIV_H__ */ -+--- /dev/null -++++ b/include/net/xdp.h -+@@ -0,0 +1,184 @@ -++/* SPDX-License-Identifier: GPL-2.0-only */ -++/* include/net/xdp.h -++ * -++ * Copyright (c) 2017 Jesper Dangaard Brouer, Red Hat Inc. -++ */ -++#ifndef __LINUX_NET_XDP_H__ -++#define __LINUX_NET_XDP_H__ -++ -++/** -++ * DOC: XDP RX-queue information -++ * -++ * The XDP RX-queue info (xdp_rxq_info) is associated with the driver -++ * level RX-ring queues. It is information that is specific to how -++ * the driver have configured a given RX-ring queue. -++ * -++ * Each xdp_buff frame received in the driver carry a (pointer) -++ * reference to this xdp_rxq_info structure. This provides the XDP -++ * data-path read-access to RX-info for both kernel and bpf-side -++ * (limited subset). -++ * -++ * For now, direct access is only safe while running in NAPI/softirq -++ * context. Contents is read-mostly and must not be updated during -++ * driver NAPI/softirq poll. -++ * -++ * The driver usage API is a register and unregister API. -++ * -++ * The struct is not directly tied to the XDP prog. A new XDP prog -++ * can be attached as long as it doesn't change the underlying -++ * RX-ring. If the RX-ring does change significantly, the NIC driver -++ * naturally need to stop the RX-ring before purging and reallocating -++ * memory. In that process the driver MUST call unregistor (which -++ * also apply for driver shutdown and unload). The register API is -++ * also mandatory during RX-ring setup. -++ */ -++ -++enum xdp_mem_type { -++ MEM_TYPE_PAGE_SHARED = 0, /* Split-page refcnt based model */ -++ MEM_TYPE_PAGE_ORDER0, /* Orig XDP full page model */ -++ MEM_TYPE_PAGE_POOL, -++ MEM_TYPE_ZERO_COPY, -++ MEM_TYPE_MAX, -++}; -++ -++/* XDP flags for ndo_xdp_xmit */ -++#define XDP_XMIT_FLUSH (1U << 0) /* doorbell signal consumer */ -++#define XDP_XMIT_FLAGS_MASK XDP_XMIT_FLUSH -++ -++struct xdp_mem_info { -++ u32 type; /* enum xdp_mem_type, but known size type */ -++ u32 id; -++}; -++ -++struct page_pool; -++ -++struct zero_copy_allocator { -++ void (*free)(struct zero_copy_allocator *zca, unsigned long handle); -++}; -++ -++struct xdp_rxq_info { -++ struct net_device *dev; -++ u32 queue_index; -++ u32 reg_state; -++ struct xdp_mem_info mem; -++} ____cacheline_aligned; /* perf critical, avoid false-sharing */ -++ -++struct xdp_buff { -++ void *data; -++ void *data_end; -++ void *data_meta; -++ void *data_hard_start; -++ unsigned long handle; -++ struct xdp_rxq_info *rxq; -++}; -++ -++struct xdp_frame { -++ void *data; -++ u16 len; -++ u16 headroom; -++ u16 metasize; -++ /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, -++ * while mem info is valid on remote CPU. -++ */ -++ struct xdp_mem_info mem; -++ struct net_device *dev_rx; /* used by cpumap */ -++}; -++ -++/* Clear kernel pointers in xdp_frame */ -++static inline void xdp_scrub_frame(struct xdp_frame *frame) -++{ -++ frame->data = NULL; -++ frame->dev_rx = NULL; -++} -++ -++struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp); -++ -++/* Convert xdp_buff to xdp_frame */ -++static inline -++struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp) -++{ -++ struct xdp_frame *xdp_frame; -++ int metasize; -++ int headroom; -++ -++ if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY) -++ return xdp_convert_zc_to_xdp_frame(xdp); -++ -++ /* Assure headroom is available for storing info */ -++ headroom = xdp->data - xdp->data_hard_start; -++ metasize = xdp->data - xdp->data_meta; -++ metasize = metasize > 0 ? metasize : 0; -++ if (unlikely((headroom - metasize) < sizeof(*xdp_frame))) -++ return NULL; -++ -++ /* Store info in top of packet */ -++ xdp_frame = xdp->data_hard_start; -++ -++ xdp_frame->data = xdp->data; -++ xdp_frame->len = xdp->data_end - xdp->data; -++ xdp_frame->headroom = headroom - sizeof(*xdp_frame); -++ xdp_frame->metasize = metasize; -++ -++ /* rxq only valid until napi_schedule ends, convert to xdp_mem_info */ -++ xdp_frame->mem = xdp->rxq->mem; -++ -++ return xdp_frame; -++} -++ -++void xdp_return_frame(struct xdp_frame *xdpf); -++void xdp_return_frame_rx_napi(struct xdp_frame *xdpf); -++void xdp_return_buff(struct xdp_buff *xdp); -++ -++/* When sending xdp_frame into the network stack, then there is no -++ * return point callback, which is needed to release e.g. DMA-mapping -++ * resources with page_pool. Thus, have explicit function to release -++ * frame resources. -++ */ -++void __xdp_release_frame(void *data, struct xdp_mem_info *mem); -++static inline void xdp_release_frame(struct xdp_frame *xdpf) -++{ -++ struct xdp_mem_info *mem = &xdpf->mem; -++ -++ /* Curr only page_pool needs this */ -++ if (mem->type == MEM_TYPE_PAGE_POOL) -++ __xdp_release_frame(xdpf->data, mem); -++} -++ -++int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, -++ struct net_device *dev, u32 queue_index); -++void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); -++void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); -++bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); -++int xdp_rxq_info_reg_mem_model(struct xdp_rxq_info *xdp_rxq, -++ enum xdp_mem_type type, void *allocator); -++void xdp_rxq_info_unreg_mem_model(struct xdp_rxq_info *xdp_rxq); -++ -++/* Drivers not supporting XDP metadata can use this helper, which -++ * rejects any room expansion for metadata as a result. -++ */ -++static __always_inline void -++xdp_set_data_meta_invalid(struct xdp_buff *xdp) -++{ -++ xdp->data_meta = xdp->data + 1; -++} -++ -++static __always_inline bool -++xdp_data_meta_unsupported(const struct xdp_buff *xdp) -++{ -++ return unlikely(xdp->data_meta > xdp->data); -++} -++ -++struct xdp_attachment_info { -++ struct bpf_prog *prog; -++ u32 flags; -++}; -++ -++struct netdev_bpf; -++int xdp_attachment_query(struct xdp_attachment_info *info, -++ struct netdev_bpf *bpf); -++bool xdp_attachment_flags_ok(struct xdp_attachment_info *info, -++ struct netdev_bpf *bpf); -++void xdp_attachment_setup(struct xdp_attachment_info *info, -++ struct netdev_bpf *bpf); -++ -++#endif /* __LINUX_NET_XDP_H__ */ -+--- a/include/linux/atomic.h -++++ b/include/linux/atomic.h -+@@ -437,6 +437,8 @@ static inline int atomic_add_unless(atom -+ return __atomic_add_unless(v, a, u) != u; -+ } -+ -++#define atomic_fetch_add_unless __atomic_add_unless -++ -+ /** -+ * atomic_inc_not_zero - increment unless the number is zero -+ * @v: pointer of type atomic_t -+--- a/include/linux/kernel.h -++++ b/include/linux/kernel.h -+@@ -45,6 +45,13 @@ -+ -+ #define STACK_MAGIC 0xdeadbeef -+ -++#define u64_to_user_ptr(x) ( \ -++ { \ -++ typecheck(u64, (x)); \ -++ (void __user *)(uintptr_t)(x); \ -++ } \ -++) -++ -+ #define REPEAT_BYTE(x) ((~0ul / 0xff) * (x)) -+ -+ #define ALIGN(x, a) __ALIGN_KERNEL((x), (a)) -+--- /dev/null -++++ b/include/linux/tnum.h -+@@ -0,0 +1,89 @@ -++/* tnum: tracked (or tristate) numbers -++ * -++ * A tnum tracks knowledge about the bits of a value. Each bit can be either -++ * known (0 or 1), or unknown (x). Arithmetic operations on tnums will -++ * propagate the unknown bits such that the tnum result represents all the -++ * possible results for possible values of the operands. -++ */ -++ -++#ifndef _LINUX_TNUM_H -++#define _LINUX_TNUM_H -++ -++#include -++ -++struct tnum { -++ u64 value; -++ u64 mask; -++}; -++ -++/* Constructors */ -++/* Represent a known constant as a tnum. */ -++struct tnum tnum_const(u64 value); -++/* A completely unknown value */ -++extern const struct tnum tnum_unknown; -++/* A value that's unknown except that @min <= value <= @max */ -++struct tnum tnum_range(u64 min, u64 max); -++ -++/* Arithmetic and logical ops */ -++/* Shift a tnum left (by a fixed shift) */ -++struct tnum tnum_lshift(struct tnum a, u8 shift); -++/* Shift (rsh) a tnum right (by a fixed shift) */ -++struct tnum tnum_rshift(struct tnum a, u8 shift); -++/* Shift (arsh) a tnum right (by a fixed min_shift) */ -++struct tnum tnum_arshift(struct tnum a, u8 min_shift, u8 insn_bitness); -++/* Add two tnums, return @a + @b */ -++struct tnum tnum_add(struct tnum a, struct tnum b); -++/* Subtract two tnums, return @a - @b */ -++struct tnum tnum_sub(struct tnum a, struct tnum b); -++/* Bitwise-AND, return @a & @b */ -++struct tnum tnum_and(struct tnum a, struct tnum b); -++/* Bitwise-OR, return @a | @b */ -++struct tnum tnum_or(struct tnum a, struct tnum b); -++/* Bitwise-XOR, return @a ^ @b */ -++struct tnum tnum_xor(struct tnum a, struct tnum b); -++/* Multiply two tnums, return @a * @b */ -++struct tnum tnum_mul(struct tnum a, struct tnum b); -++ -++/* Return a tnum representing numbers satisfying both @a and @b */ -++struct tnum tnum_intersect(struct tnum a, struct tnum b); -++ -++/* Return @a with all but the lowest @size bytes cleared */ -++struct tnum tnum_cast(struct tnum a, u8 size); -++ -++/* Returns true if @a is a known constant */ -++static inline bool tnum_is_const(struct tnum a) -++{ -++ return !a.mask; -++} -++ -++/* Returns true if @a == tnum_const(@b) */ -++static inline bool tnum_equals_const(struct tnum a, u64 b) -++{ -++ return tnum_is_const(a) && a.value == b; -++} -++ -++/* Returns true if @a is completely unknown */ -++static inline bool tnum_is_unknown(struct tnum a) -++{ -++ return !~a.mask; -++} -++ -++/* Returns true if @a is known to be a multiple of @size. -++ * @size must be a power of two. -++ */ -++bool tnum_is_aligned(struct tnum a, u64 size); -++ -++/* Returns true if @b represents a subset of @a. */ -++bool tnum_in(struct tnum a, struct tnum b); -++ -++/* Formatting functions. These have snprintf-like semantics: they will write -++ * up to @size bytes (including the terminating NUL byte), and return the number -++ * of bytes (excluding the terminating NUL) which would have been written had -++ * sufficient space been available. (Thus tnum_sbin always returns 64.) -++ */ -++/* Format a tnum as a pair of hex numbers (value; mask) */ -++int tnum_strn(char *str, size_t size, struct tnum a); -++/* Format a tnum as tristate binary expansion */ -++int tnum_sbin(char *str, size_t size, struct tnum a); -++ -++#endif /* _LINUX_TNUM_H */ -+--- a/include/linux/bitmap.h -++++ b/include/linux/bitmap.h -+@@ -326,6 +326,24 @@ static inline int bitmap_parse(const cha -+ return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); -+ } -+ -++/** -++ * bitmap_from_u64 - Check and swap words within u64. -++ * @mask: source bitmap -++ * @dst: destination bitmap -++ * -++ * In 32-bit Big Endian kernel, when using ``(u32 *)(&val)[*]`` -++ * to read u64 mask, we will get the wrong word. -++ * That is ``(u32 *)(&val)[0]`` gets the upper 32 bits, -++ * but we expect the lower 32-bits of u64. -++ */ -++static inline void bitmap_from_u64(unsigned long *dst, u64 mask) -++{ -++ dst[0] = mask & ULONG_MAX; -++ -++ if (sizeof(mask) > sizeof(unsigned long)) -++ dst[1] = mask >> 32; -++} -++ -+ #endif /* __ASSEMBLY__ */ -+ -+ #endif /* __LINUX_BITMAP_H */ -+--- /dev/null -++++ b/include/linux/overflow.h -+@@ -0,0 +1,320 @@ -++/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -++#ifndef __LINUX_OVERFLOW_H -++#define __LINUX_OVERFLOW_H -++ -++#include -++#include -++ -++/* -++ * In the fallback code below, we need to compute the minimum and -++ * maximum values representable in a given type. These macros may also -++ * be useful elsewhere, so we provide them outside the -++ * COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW block. -++ * -++ * It would seem more obvious to do something like -++ * -++ * #define type_min(T) (T)(is_signed_type(T) ? (T)1 << (8*sizeof(T)-1) : 0) -++ * #define type_max(T) (T)(is_signed_type(T) ? ((T)1 << (8*sizeof(T)-1)) - 1 : ~(T)0) -++ * -++ * Unfortunately, the middle expressions, strictly speaking, have -++ * undefined behaviour, and at least some versions of gcc warn about -++ * the type_max expression (but not if -fsanitize=undefined is in -++ * effect; in that case, the warning is deferred to runtime...). -++ * -++ * The slightly excessive casting in type_min is to make sure the -++ * macros also produce sensible values for the exotic type _Bool. [The -++ * overflow checkers only almost work for _Bool, but that's -++ * a-feature-not-a-bug, since people shouldn't be doing arithmetic on -++ * _Bools. Besides, the gcc builtins don't allow _Bool* as third -++ * argument.] -++ * -++ * Idea stolen from -++ * https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html - -++ * credit to Christian Biere. -++ */ -++#define is_signed_type(type) (((type)(-1)) < (type)1) -++#define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type))) -++#define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) -++#define type_min(T) ((T)((T)-type_max(T)-(T)1)) -++ -++/* -++ * Avoids triggering -Wtype-limits compilation warning, -++ * while using unsigned data types to check a < 0. -++ */ -++#define is_non_negative(a) ((a) > 0 || (a) == 0) -++#define is_negative(a) (!(is_non_negative(a))) -++ -++#ifdef COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW -++/* -++ * For simplicity and code hygiene, the fallback code below insists on -++ * a, b and *d having the same type (similar to the min() and max() -++ * macros), whereas gcc's type-generic overflow checkers accept -++ * different types. Hence we don't just make check_add_overflow an -++ * alias for __builtin_add_overflow, but add type checks similar to -++ * below. -++ */ -++#define check_add_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ __builtin_add_overflow(__a, __b, __d); \ -++}) -++ -++#define check_sub_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ __builtin_sub_overflow(__a, __b, __d); \ -++}) -++ -++#define check_mul_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ __builtin_mul_overflow(__a, __b, __d); \ -++}) -++ -++#else -++ -++ -++/* Checking for unsigned overflow is relatively easy without causing UB. */ -++#define __unsigned_add_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = __a + __b; \ -++ *__d < __a; \ -++}) -++#define __unsigned_sub_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = __a - __b; \ -++ __a < __b; \ -++}) -++/* -++ * If one of a or b is a compile-time constant, this avoids a division. -++ */ -++#define __unsigned_mul_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = __a * __b; \ -++ __builtin_constant_p(__b) ? \ -++ __b > 0 && __a > type_max(typeof(__a)) / __b : \ -++ __a > 0 && __b > type_max(typeof(__b)) / __a; \ -++}) -++ -++/* -++ * For signed types, detecting overflow is much harder, especially if -++ * we want to avoid UB. But the interface of these macros is such that -++ * we must provide a result in *d, and in fact we must produce the -++ * result promised by gcc's builtins, which is simply the possibly -++ * wrapped-around value. Fortunately, we can just formally do the -++ * operations in the widest relevant unsigned type (u64) and then -++ * truncate the result - gcc is smart enough to generate the same code -++ * with and without the (u64) casts. -++ */ -++ -++/* -++ * Adding two signed integers can overflow only if they have the same -++ * sign, and overflow has happened iff the result has the opposite -++ * sign. -++ */ -++#define __signed_add_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = (u64)__a + (u64)__b; \ -++ (((~(__a ^ __b)) & (*__d ^ __a)) \ -++ & type_min(typeof(__a))) != 0; \ -++}) -++ -++/* -++ * Subtraction is similar, except that overflow can now happen only -++ * when the signs are opposite. In this case, overflow has happened if -++ * the result has the opposite sign of a. -++ */ -++#define __signed_sub_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = (u64)__a - (u64)__b; \ -++ ((((__a ^ __b)) & (*__d ^ __a)) \ -++ & type_min(typeof(__a))) != 0; \ -++}) -++ -++/* -++ * Signed multiplication is rather hard. gcc always follows C99, so -++ * division is truncated towards 0. This means that we can write the -++ * overflow check like this: -++ * -++ * (a > 0 && (b > MAX/a || b < MIN/a)) || -++ * (a < -1 && (b > MIN/a || b < MAX/a) || -++ * (a == -1 && b == MIN) -++ * -++ * The redundant casts of -1 are to silence an annoying -Wtype-limits -++ * (included in -Wextra) warning: When the type is u8 or u16, the -++ * __b_c_e in check_mul_overflow obviously selects -++ * __unsigned_mul_overflow, but unfortunately gcc still parses this -++ * code and warns about the limited range of __b. -++ */ -++ -++#define __signed_mul_overflow(a, b, d) ({ \ -++ typeof(a) __a = (a); \ -++ typeof(b) __b = (b); \ -++ typeof(d) __d = (d); \ -++ typeof(a) __tmax = type_max(typeof(a)); \ -++ typeof(a) __tmin = type_min(typeof(a)); \ -++ (void) (&__a == &__b); \ -++ (void) (&__a == __d); \ -++ *__d = (u64)__a * (u64)__b; \ -++ (__b > 0 && (__a > __tmax/__b || __a < __tmin/__b)) || \ -++ (__b < (typeof(__b))-1 && (__a > __tmin/__b || __a < __tmax/__b)) || \ -++ (__b == (typeof(__b))-1 && __a == __tmin); \ -++}) -++ -++ -++#define check_add_overflow(a, b, d) \ -++ __builtin_choose_expr(is_signed_type(typeof(a)), \ -++ __signed_add_overflow(a, b, d), \ -++ __unsigned_add_overflow(a, b, d)) -++ -++#define check_sub_overflow(a, b, d) \ -++ __builtin_choose_expr(is_signed_type(typeof(a)), \ -++ __signed_sub_overflow(a, b, d), \ -++ __unsigned_sub_overflow(a, b, d)) -++ -++#define check_mul_overflow(a, b, d) \ -++ __builtin_choose_expr(is_signed_type(typeof(a)), \ -++ __signed_mul_overflow(a, b, d), \ -++ __unsigned_mul_overflow(a, b, d)) -++ -++ -++#endif /* COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW */ -++ -++/** check_shl_overflow() - Calculate a left-shifted value and check overflow -++ * -++ * @a: Value to be shifted -++ * @s: How many bits left to shift -++ * @d: Pointer to where to store the result -++ * -++ * Computes *@d = (@a << @s) -++ * -++ * Returns true if '*d' cannot hold the result or when 'a << s' doesn't -++ * make sense. Example conditions: -++ * - 'a << s' causes bits to be lost when stored in *d. -++ * - 's' is garbage (e.g. negative) or so large that the result of -++ * 'a << s' is guaranteed to be 0. -++ * - 'a' is negative. -++ * - 'a << s' sets the sign bit, if any, in '*d'. -++ * -++ * '*d' will hold the results of the attempted shift, but is not -++ * considered "safe for use" if false is returned. -++ */ -++#define check_shl_overflow(a, s, d) ({ \ -++ typeof(a) _a = a; \ -++ typeof(s) _s = s; \ -++ typeof(d) _d = d; \ -++ u64 _a_full = _a; \ -++ unsigned int _to_shift = \ -++ is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0; \ -++ *_d = (_a_full << _to_shift); \ -++ (_to_shift != _s || is_negative(*_d) || is_negative(_a) || \ -++ (*_d >> _to_shift) != _a); \ -++}) -++ -++/** -++ * array_size() - Calculate size of 2-dimensional array. -++ * -++ * @a: dimension one -++ * @b: dimension two -++ * -++ * Calculates size of 2-dimensional array: @a * @b. -++ * -++ * Returns: number of bytes needed to represent the array or SIZE_MAX on -++ * overflow. -++ */ -++static inline __must_check size_t array_size(size_t a, size_t b) -++{ -++ size_t bytes; -++ -++ if (check_mul_overflow(a, b, &bytes)) -++ return SIZE_MAX; -++ -++ return bytes; -++} -++ -++/** -++ * array3_size() - Calculate size of 3-dimensional array. -++ * -++ * @a: dimension one -++ * @b: dimension two -++ * @c: dimension three -++ * -++ * Calculates size of 3-dimensional array: @a * @b * @c. -++ * -++ * Returns: number of bytes needed to represent the array or SIZE_MAX on -++ * overflow. -++ */ -++static inline __must_check size_t array3_size(size_t a, size_t b, size_t c) -++{ -++ size_t bytes; -++ -++ if (check_mul_overflow(a, b, &bytes)) -++ return SIZE_MAX; -++ if (check_mul_overflow(bytes, c, &bytes)) -++ return SIZE_MAX; -++ -++ return bytes; -++} -++ -++/* -++ * Compute a*b+c, returning SIZE_MAX on overflow. Internal helper for -++ * struct_size() below. -++ */ -++static inline __must_check size_t __ab_c_size(size_t a, size_t b, size_t c) -++{ -++ size_t bytes; -++ -++ if (check_mul_overflow(a, b, &bytes)) -++ return SIZE_MAX; -++ if (check_add_overflow(bytes, c, &bytes)) -++ return SIZE_MAX; -++ -++ return bytes; -++} -++ -++/** -++ * struct_size() - Calculate size of structure with trailing array. -++ * @p: Pointer to the structure. -++ * @member: Name of the array member. -++ * @n: Number of elements in the array. -++ * -++ * Calculates size of memory needed for structure @p followed by an -++ * array of @n @member elements. -++ * -++ * Return: number of bytes needed or SIZE_MAX on overflow. -++ */ -++#define struct_size(p, member, n) \ -++ __ab_c_size(n, \ -++ sizeof(*(p)->member) + __must_be_array((p)->member),\ -++ sizeof(*(p))) -++ -++#endif /* __LINUX_OVERFLOW_H */ -+--- a/net/core/filter.c -++++ b/net/core/filter.c -+@@ -1,3 +1,4 @@ -++// SPDX-License-Identifier: GPL-2.0-or-later -+ /* -+ * Linux Socket Filter - Kernel level socket filtering -+ * -+@@ -12,11 +13,6 @@ -+ * Alexei Starovoitov -+ * Daniel Borkmann -+ * -+- * This program is free software; you can redistribute it and/or -+- * modify it under the terms of the GNU General Public License -+- * as published by the Free Software Foundation; either version -+- * 2 of the License, or (at your option) any later version. -+- * -+ * Andi Kleen - Fix a few bad bugs and races. -+ * Kris Katterjohn - Added many additional checks in bpf_check_classic() -+ */ -+@@ -26,11 +22,14 @@ -+ #include -+ #include -+ #include -++#include -+ #include -+ #include -+ #include -+ #include -++#include -+ #include -++#include -+ #include -+ #include -+ #include -+@@ -39,17 +38,32 @@ -+ #include -+ #include -+ #include -+-#include -++#include -+ #include -++#include -+ #include -+ #include -+ #include -+ #include -+ #include -+ #include -+-#include -+ #include -+ #include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -++#include -+ -+ /** -+ * sk_filter_trim_cap - run a packet through a socket filter -+@@ -84,7 +98,12 @@ int sk_filter_trim_cap(struct sock *sk, -+ rcu_read_lock(); -+ filter = rcu_dereference(sk->sk_filter); -+ if (filter) { -+- unsigned int pkt_len = bpf_prog_run_save_cb(filter->prog, skb); -++ struct sock *save_sk = skb->sk; -++ unsigned int pkt_len; -++ -++ skb->sk = sk; -++ pkt_len = bpf_prog_run_save_cb(filter->prog, skb); -++ skb->sk = save_sk; -+ err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM; -+ } -+ rcu_read_unlock(); -+@@ -93,14 +112,13 @@ int sk_filter_trim_cap(struct sock *sk, -+ } -+ EXPORT_SYMBOL(sk_filter_trim_cap); -+ -+-static u64 __skb_get_pay_offset(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) -++BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb) -+ { -+- return skb_get_poff((struct sk_buff *)(unsigned long) ctx); -++ return skb_get_poff(skb); -+ } -+ -+-static u64 __skb_get_nlattr(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) -++BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x) -+ { -+- struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; -+ struct nlattr *nla; -+ -+ if (skb_is_nonlinear(skb)) -+@@ -119,9 +137,8 @@ static u64 __skb_get_nlattr(u64 ctx, u64 -+ return 0; -+ } -+ -+-static u64 __skb_get_nlattr_nest(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) -++BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x) -+ { -+- struct sk_buff *skb = (struct sk_buff *)(unsigned long) ctx; -+ struct nlattr *nla; -+ -+ if (skb_is_nonlinear(skb)) -+@@ -144,11 +161,98 @@ static u64 __skb_get_nlattr_nest(u64 ctx -+ return 0; -+ } -+ -+-static u64 __get_raw_cpu_id(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) -++BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *, -++ data, int, headlen, int, offset) -++{ -++ u8 tmp, *ptr; -++ const int len = sizeof(tmp); -++ -++ if (offset >= 0) { -++ if (headlen - offset >= len) -++ return *(u8 *)(data + offset); -++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) -++ return tmp; -++ } else { -++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); -++ if (likely(ptr)) -++ return *(u8 *)ptr; -++ } -++ -++ return -EFAULT; -++} -++ -++BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb, -++ int, offset) -++{ -++ return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len, -++ offset); -++} -++ -++BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *, -++ data, int, headlen, int, offset) -++{ -++ u16 tmp, *ptr; -++ const int len = sizeof(tmp); -++ -++ if (offset >= 0) { -++ if (headlen - offset >= len) -++ return get_unaligned_be16(data + offset); -++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) -++ return be16_to_cpu(tmp); -++ } else { -++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); -++ if (likely(ptr)) -++ return get_unaligned_be16(ptr); -++ } -++ -++ return -EFAULT; -++} -++ -++BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb, -++ int, offset) -++{ -++ return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len, -++ offset); -++} -++ -++BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *, -++ data, int, headlen, int, offset) -++{ -++ u32 tmp, *ptr; -++ const int len = sizeof(tmp); -++ -++ if (likely(offset >= 0)) { -++ if (headlen - offset >= len) -++ return get_unaligned_be32(data + offset); -++ if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp))) -++ return be32_to_cpu(tmp); -++ } else { -++ ptr = bpf_internal_load_pointer_neg_helper(skb, offset, len); -++ if (likely(ptr)) -++ return get_unaligned_be32(ptr); -++ } -++ -++ return -EFAULT; -++} -++ -++BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb, -++ int, offset) -++{ -++ return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len, -++ offset); -++} -++ -++BPF_CALL_0(bpf_get_raw_cpu_id) -+ { -+ return raw_smp_processor_id(); -+ } -+ -++static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { -++ .func = bpf_get_raw_cpu_id, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++}; -++ -+ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, -+ struct bpf_insn *insn_buf) -+ { -+@@ -178,22 +282,18 @@ static u32 convert_skb_access(int skb_fi -+ break; -+ -+ case SKF_AD_VLAN_TAG: -+- case SKF_AD_VLAN_TAG_PRESENT: -+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); -+- BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); -+ -+ /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */ -+ *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, -+ offsetof(struct sk_buff, vlan_tci)); -+- if (skb_field == SKF_AD_VLAN_TAG) { -+- *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, -+- ~VLAN_TAG_PRESENT); -+- } else { -+- /* dst_reg >>= 12 */ -+- *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 12); -+- /* dst_reg &= 1 */ -++ break; -++ case SKF_AD_VLAN_TAG_PRESENT: -++ *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET()); -++ if (PKT_VLAN_PRESENT_BIT) -++ *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT); -++ if (PKT_VLAN_PRESENT_BIT < 7) -+ *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, 1); -+- } -+ break; -+ } -+ -+@@ -226,9 +326,8 @@ static bool convert_bpf_extensions(struc -+ case SKF_AD_OFF + SKF_AD_HATYPE: -+ BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); -+ BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); -+- BUILD_BUG_ON(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)) < 0); -+ -+- *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), -+ BPF_REG_TMP, BPF_REG_CTX, -+ offsetof(struct sk_buff, dev)); -+ /* if (tmp != 0) goto pc + 1 */ -+@@ -295,16 +394,16 @@ static bool convert_bpf_extensions(struc -+ /* Emit call(arg1=CTX, arg2=A, arg3=X) */ -+ switch (fp->k) { -+ case SKF_AD_OFF + SKF_AD_PAY_OFFSET: -+- *insn = BPF_EMIT_CALL(__skb_get_pay_offset); -++ *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset); -+ break; -+ case SKF_AD_OFF + SKF_AD_NLATTR: -+- *insn = BPF_EMIT_CALL(__skb_get_nlattr); -++ *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr); -+ break; -+ case SKF_AD_OFF + SKF_AD_NLATTR_NEST: -+- *insn = BPF_EMIT_CALL(__skb_get_nlattr_nest); -++ *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest); -+ break; -+ case SKF_AD_OFF + SKF_AD_CPU: -+- *insn = BPF_EMIT_CALL(__get_raw_cpu_id); -++ *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id); -+ break; -+ case SKF_AD_OFF + SKF_AD_RANDOM: -+ *insn = BPF_EMIT_CALL(bpf_user_rnd_u32); -+@@ -331,35 +430,101 @@ static bool convert_bpf_extensions(struc -+ return true; -+ } -+ -++static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp) -++{ -++ const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS); -++ int size = bpf_size_to_bytes(BPF_SIZE(fp->code)); -++ bool endian = BPF_SIZE(fp->code) == BPF_H || -++ BPF_SIZE(fp->code) == BPF_W; -++ bool indirect = BPF_MODE(fp->code) == BPF_IND; -++ const int ip_align = NET_IP_ALIGN; -++ struct bpf_insn *insn = *insnp; -++ int offset = fp->k; -++ -++ if (!indirect && -++ ((unaligned_ok && offset >= 0) || -++ (!unaligned_ok && offset >= 0 && -++ offset + ip_align >= 0 && -++ offset + ip_align % size == 0))) { -++ bool ldx_off_ok = offset <= S16_MAX; -++ -++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H); -++ if (offset) -++ *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset); -++ *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP, -++ size, 2 + endian + (!ldx_off_ok * 2)); -++ if (ldx_off_ok) { -++ *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, -++ BPF_REG_D, offset); -++ } else { -++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D); -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset); -++ *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A, -++ BPF_REG_TMP, 0); -++ } -++ if (endian) -++ *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8); -++ *insn++ = BPF_JMP_A(8); -++ } -++ -++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX); -++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D); -++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H); -++ if (!indirect) { -++ *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset); -++ } else { -++ *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X); -++ if (fp->k) -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset); -++ } -++ -++ switch (BPF_SIZE(fp->code)) { -++ case BPF_B: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8); -++ break; -++ case BPF_H: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16); -++ break; -++ case BPF_W: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32); -++ break; -++ default: -++ return false; -++ } -++ -++ *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2); -++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); -++ *insn = BPF_EXIT_INSN(); -++ -++ *insnp = insn; -++ return true; -++} -++ -+ /** -+ * bpf_convert_filter - convert filter program -+ * @prog: the user passed filter program -+ * @len: the length of the user passed filter program -+- * @new_prog: buffer where converted program will be stored -++ * @new_prog: allocated 'struct bpf_prog' or NULL -+ * @new_len: pointer to store length of converted program -++ * @seen_ld_abs: bool whether we've seen ld_abs/ind -+ * -+- * Remap 'sock_filter' style BPF instruction set to 'sock_filter_ext' style. -++ * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn' -++ * style extended BPF (eBPF). -+ * Conversion workflow: -+ * -+ * 1) First pass for calculating the new program length: -+- * bpf_convert_filter(old_prog, old_len, NULL, &new_len) -++ * bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs) -+ * -+ * 2) 2nd pass to remap in two passes: 1st pass finds new -+ * jump offsets, 2nd pass remapping: -+- * new_prog = kmalloc(sizeof(struct bpf_insn) * new_len); -+- * bpf_convert_filter(old_prog, old_len, new_prog, &new_len); -+- * -+- * User BPF's register A is mapped to our BPF register 6, user BPF -+- * register X is mapped to BPF register 7; frame pointer is always -+- * register 10; Context 'void *ctx' is stored in register 1, that is, -+- * for socket filters: ctx == 'struct sk_buff *', for seccomp: -+- * ctx == 'struct seccomp_data *'. -++ * bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs) -+ */ -+ static int bpf_convert_filter(struct sock_filter *prog, int len, -+- struct bpf_insn *new_prog, int *new_len) -++ struct bpf_prog *new_prog, int *new_len, -++ bool *seen_ld_abs) -+ { -+- int new_flen = 0, pass = 0, target, i; -+- struct bpf_insn *new_insn; -++ int new_flen = 0, pass = 0, target, i, stack_off; -++ struct bpf_insn *new_insn, *first_insn = NULL; -+ struct sock_filter *fp; -+ int *addrs = NULL; -+ u8 bpf_src; -+@@ -371,6 +536,7 @@ static int bpf_convert_filter(struct soc -+ return -EINVAL; -+ -+ if (new_prog) { -++ first_insn = new_prog->insnsi; -+ addrs = kcalloc(len, sizeof(*addrs), -+ GFP_KERNEL | __GFP_NOWARN); -+ if (!addrs) -+@@ -378,19 +544,47 @@ static int bpf_convert_filter(struct soc -+ } -+ -+ do_pass: -+- new_insn = new_prog; -++ new_insn = first_insn; -+ fp = prog; -+ -+- if (new_insn) -+- *new_insn = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); -+- new_insn++; -++ /* Classic BPF related prologue emission. */ -++ if (new_prog) { -++ /* Classic BPF expects A and X to be reset first. These need -++ * to be guaranteed to be the first two instructions. -++ */ -++ *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); -++ *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X); -++ -++ /* All programs must keep CTX in callee saved BPF_REG_CTX. -++ * In eBPF case it's done by the compiler, here we need to -++ * do this ourself. Initial CTX is present in BPF_REG_ARG1. -++ */ -++ *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1); -++ if (*seen_ld_abs) { -++ /* For packet access in classic BPF, cache skb->data -++ * in callee-saved BPF R8 and skb->len - skb->data_len -++ * (headlen) in BPF R9. Since classic BPF is read-only -++ * on CTX, we only need to cache it once. -++ */ -++ *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), -++ BPF_REG_D, BPF_REG_CTX, -++ offsetof(struct sk_buff, data)); -++ *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX, -++ offsetof(struct sk_buff, len)); -++ *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX, -++ offsetof(struct sk_buff, data_len)); -++ *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP); -++ } -++ } else { -++ new_insn += 3; -++ } -+ -+ for (i = 0; i < len; fp++, i++) { -+- struct bpf_insn tmp_insns[6] = { }; -++ struct bpf_insn tmp_insns[32] = { }; -+ struct bpf_insn *insn = tmp_insns; -+ -+ if (addrs) -+- addrs[i] = new_insn - new_prog; -++ addrs[i] = new_insn - first_insn; -+ -+ switch (fp->code) { -+ /* All arithmetic insns and skb loads map as-is. */ -+@@ -429,6 +623,22 @@ do_pass: -+ BPF_MODE(fp->code) == BPF_ABS && -+ convert_bpf_extensions(fp, &insn)) -+ break; -++ if (BPF_CLASS(fp->code) == BPF_LD && -++ convert_bpf_ld_abs(fp, &insn)) { -++ *seen_ld_abs = true; -++ break; -++ } -++ -++ if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) || -++ fp->code == (BPF_ALU | BPF_MOD | BPF_X)) { -++ *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X); -++ /* Error with exception code on div/mod by 0. -++ * For cBPF programs, this was always return 0. -++ */ -++ *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2); -++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A); -++ *insn++ = BPF_EXIT_INSN(); -++ } -+ -+ *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k); -+ break; -+@@ -441,11 +651,18 @@ do_pass: -+ -+ #define BPF_EMIT_JMP \ -+ do { \ -++ const s32 off_min = S16_MIN, off_max = S16_MAX; \ -++ s32 off; \ -++ \ -+ if (target >= len || target < 0) \ -+ goto err; \ -+- insn->off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ -++ off = addrs ? addrs[target] - addrs[i] - 1 : 0; \ -+ /* Adjust pc relative offset for 2nd or 3rd insn. */ \ -+- insn->off -= insn - tmp_insns; \ -++ off -= insn - tmp_insns; \ -++ /* Reject anything not fitting into insn->off. */ \ -++ if (off < off_min || off > off_max) \ -++ goto err; \ -++ insn->off = off; \ -+ } while (0) -+ -+ case BPF_JMP | BPF_JA: -+@@ -487,14 +704,27 @@ do_pass: -+ break; -+ } -+ -+- /* Convert JEQ into JNE when 'jump_true' is next insn. */ -+- if (fp->jt == 0 && BPF_OP(fp->code) == BPF_JEQ) { -+- insn->code = BPF_JMP | BPF_JNE | bpf_src; -++ /* Convert some jumps when 'jump_true' is next insn. */ -++ if (fp->jt == 0) { -++ switch (BPF_OP(fp->code)) { -++ case BPF_JEQ: -++ insn->code = BPF_JMP | BPF_JNE | bpf_src; -++ break; -++ case BPF_JGT: -++ insn->code = BPF_JMP | BPF_JLE | bpf_src; -++ break; -++ case BPF_JGE: -++ insn->code = BPF_JMP | BPF_JLT | bpf_src; -++ break; -++ default: -++ goto jmp_rest; -++ } -++ -+ target = i + fp->jf + 1; -+ BPF_EMIT_JMP; -+ break; -+ } -+- -++jmp_rest: -+ /* Other jumps are mapped into two insns: Jxx and JA. */ -+ target = i + fp->jt + 1; -+ insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src; -+@@ -507,44 +737,64 @@ do_pass: -+ break; -+ -+ /* ldxb 4 * ([14] & 0xf) is remaped into 6 insns. */ -+- case BPF_LDX | BPF_MSH | BPF_B: -+- /* tmp = A */ -+- *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_A); -++ case BPF_LDX | BPF_MSH | BPF_B: { -++ struct sock_filter tmp = { -++ .code = BPF_LD | BPF_ABS | BPF_B, -++ .k = fp->k, -++ }; -++ -++ *seen_ld_abs = true; -++ -++ /* X = A */ -++ *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); -+ /* A = BPF_R0 = *(u8 *) (skb->data + K) */ -+- *insn++ = BPF_LD_ABS(BPF_B, fp->k); -++ convert_bpf_ld_abs(&tmp, &insn); -++ insn++; -+ /* A &= 0xf */ -+ *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf); -+ /* A <<= 2 */ -+ *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2); -++ /* tmp = X */ -++ *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X); -+ /* X = A */ -+ *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A); -+ /* A = tmp */ -+ *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP); -+ break; -+- -+- /* RET_K, RET_A are remaped into 2 insns. */ -++ } -++ /* RET_K is remaped into 2 insns. RET_A case doesn't need an -++ * extra mov as BPF_REG_0 is already mapped into BPF_REG_A. -++ */ -+ case BPF_RET | BPF_A: -+ case BPF_RET | BPF_K: -+- *insn++ = BPF_MOV32_RAW(BPF_RVAL(fp->code) == BPF_K ? -+- BPF_K : BPF_X, BPF_REG_0, -+- BPF_REG_A, fp->k); -++ if (BPF_RVAL(fp->code) == BPF_K) -++ *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0, -++ 0, fp->k); -+ *insn = BPF_EXIT_INSN(); -+ break; -+ -+ /* Store to stack. */ -+ case BPF_ST: -+ case BPF_STX: -++ stack_off = fp->k * 4 + 4; -+ *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) == -+ BPF_ST ? BPF_REG_A : BPF_REG_X, -+- -(BPF_MEMWORDS - fp->k) * 4); -++ -stack_off); -++ /* check_load_and_stores() verifies that classic BPF can -++ * load from stack only after write, so tracking -++ * stack_depth for ST|STX insns is enough -++ */ -++ if (new_prog && new_prog->aux->stack_depth < stack_off) -++ new_prog->aux->stack_depth = stack_off; -+ break; -+ -+ /* Load from stack. */ -+ case BPF_LD | BPF_MEM: -+ case BPF_LDX | BPF_MEM: -++ stack_off = fp->k * 4 + 4; -+ *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ? -+ BPF_REG_A : BPF_REG_X, BPF_REG_FP, -+- -(BPF_MEMWORDS - fp->k) * 4); -++ -stack_off); -+ break; -+ -+ /* A = K or X = K */ -+@@ -592,13 +842,15 @@ do_pass: -+ -+ if (!new_prog) { -+ /* Only calculating new length. */ -+- *new_len = new_insn - new_prog; -++ *new_len = new_insn - first_insn; -++ if (*seen_ld_abs) -++ *new_len += 4; /* Prologue bits. */ -+ return 0; -+ } -+ -+ pass++; -+- if (new_flen != new_insn - new_prog) { -+- new_flen = new_insn - new_prog; -++ if (new_flen != new_insn - first_insn) { -++ new_flen = new_insn - first_insn; -+ if (pass > 2) -+ goto err; -+ goto do_pass; -+@@ -738,6 +990,17 @@ static bool chk_code_allowed(u16 code_to -+ return codes[code_to_probe]; -+ } -+ -++static bool bpf_check_basics_ok(const struct sock_filter *filter, -++ unsigned int flen) -++{ -++ if (filter == NULL) -++ return false; -++ if (flen == 0 || flen > BPF_MAXINSNS) -++ return false; -++ -++ return true; -++} -++ -+ /** -+ * bpf_check_classic - verify socket filter code -+ * @filter: filter to verify -+@@ -758,9 +1021,6 @@ static int bpf_check_classic(const struc -+ bool anc_found; -+ int pc; -+ -+- if (flen == 0 || flen > BPF_MAXINSNS) -+- return -EINVAL; -+- -+ /* Check the filter code now */ -+ for (pc = 0; pc < flen; pc++) { -+ const struct sock_filter *ftest = &filter[pc]; -+@@ -901,7 +1161,7 @@ static void sk_filter_release_rcu(struct -+ */ -+ static void sk_filter_release(struct sk_filter *fp) -+ { -+- if (atomic_dec_and_test(&fp->refcnt)) -++ if (refcount_dec_and_test(&fp->refcnt)) -+ call_rcu(&fp->rcu, sk_filter_release_rcu); -+ } -+ -+@@ -916,25 +1176,37 @@ void sk_filter_uncharge(struct sock *sk, -+ /* try to charge the socket memory if there is space available -+ * return true on success -+ */ -+-bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) -++static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp) -+ { -+ u32 filter_size = bpf_prog_size(fp->prog->len); -+ -+ /* same check as in sock_kmalloc() */ -+ if (filter_size <= sysctl_optmem_max && -+ atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) { -+- atomic_inc(&fp->refcnt); -+ atomic_add(filter_size, &sk->sk_omem_alloc); -+ return true; -+ } -+ return false; -+ } -+ -++bool sk_filter_charge(struct sock *sk, struct sk_filter *fp) -++{ -++ if (!refcount_inc_not_zero(&fp->refcnt)) -++ return false; -++ -++ if (!__sk_filter_charge(sk, fp)) { -++ sk_filter_release(fp); -++ return false; -++ } -++ return true; -++} -++ -+ static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp) -+ { -+ struct sock_filter *old_prog; -+ struct bpf_prog *old_fp; -+ int err, new_len, old_len = fp->len; -++ bool seen_ld_abs = false; -+ -+ /* We are free to overwrite insns et al right here as it -+ * won't be used at this point in time anymore internally -+@@ -956,7 +1228,8 @@ static struct bpf_prog *bpf_migrate_filt -+ } -+ -+ /* 1st pass: calculate the new program length. */ -+- err = bpf_convert_filter(old_prog, old_len, NULL, &new_len); -++ err = bpf_convert_filter(old_prog, old_len, NULL, &new_len, -++ &seen_ld_abs); -+ if (err) -+ goto out_err_free; -+ -+@@ -975,7 +1248,8 @@ static struct bpf_prog *bpf_migrate_filt -+ fp->len = new_len; -+ -+ /* 2nd pass: remap sock_filter insns into bpf_insn insns. */ -+- err = bpf_convert_filter(old_prog, old_len, fp->insnsi, &new_len); -++ err = bpf_convert_filter(old_prog, old_len, fp, &new_len, -++ &seen_ld_abs); -+ if (err) -+ /* 2nd bpf_convert_filter() can fail only if it fails -+ * to allocate memory, remapping must succeed. Note, -+@@ -984,7 +1258,9 @@ static struct bpf_prog *bpf_migrate_filt -+ */ -+ goto out_err_free; -+ -+- bpf_prog_select_runtime(fp); -++ fp = bpf_prog_select_runtime(fp, &err); -++ if (err) -++ goto out_err_free; -+ -+ kfree(old_prog); -+ return fp; -+@@ -1051,7 +1327,7 @@ int bpf_prog_create(struct bpf_prog **pf -+ struct bpf_prog *fp; -+ -+ /* Make sure new filter is there and in the right amounts. */ -+- if (fprog->filter == NULL) -++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) -+ return -EINVAL; -+ -+ fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); -+@@ -1098,7 +1374,7 @@ int bpf_prog_create_from_user(struct bpf -+ int err; -+ -+ /* Make sure new filter is there and in the right amounts. */ -+- if (fprog->filter == NULL) -++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) -+ return -EINVAL; -+ -+ fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); -+@@ -1139,8 +1415,7 @@ void bpf_prog_destroy(struct bpf_prog *f -+ } -+ EXPORT_SYMBOL_GPL(bpf_prog_destroy); -+ -+-static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk, -+- bool locked) -++static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk) -+ { -+ struct sk_filter *fp, *old_fp; -+ -+@@ -1149,53 +1424,44 @@ static int __sk_attach_prog(struct bpf_p -+ return -ENOMEM; -+ -+ fp->prog = prog; -+- atomic_set(&fp->refcnt, 0); -+ -+- if (!sk_filter_charge(sk, fp)) { -++ if (!__sk_filter_charge(sk, fp)) { -+ kfree(fp); -+ return -ENOMEM; -+ } -++ refcount_set(&fp->refcnt, 1); -+ -+- old_fp = rcu_dereference_protected(sk->sk_filter, locked); -++ old_fp = rcu_dereference_protected(sk->sk_filter, -++ lockdep_sock_is_held(sk)); -+ rcu_assign_pointer(sk->sk_filter, fp); -++ -+ if (old_fp) -+ sk_filter_uncharge(sk, old_fp); -+ -+ return 0; -+ } -+ -+-/** -+- * sk_attach_filter - attach a socket filter -+- * @fprog: the filter program -+- * @sk: the socket to use -+- * -+- * Attach the user's filter code. We first run some sanity checks on -+- * it to make sure it does not explode on us later. If an error -+- * occurs or there is insufficient memory for the filter a negative -+- * errno code is returned. On success the return is zero. -+- */ -+-int __sk_attach_filter(struct sock_fprog *fprog, struct sock *sk, -+- bool locked) -++static -++struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk) -+ { -+ unsigned int fsize = bpf_classic_proglen(fprog); -+- unsigned int bpf_fsize = bpf_prog_size(fprog->len); -+ struct bpf_prog *prog; -+ int err; -+ -+ if (sock_flag(sk, SOCK_FILTER_LOCKED)) -+- return -EPERM; -++ return ERR_PTR(-EPERM); -+ -+ /* Make sure new filter is there and in the right amounts. */ -+- if (fprog->filter == NULL) -+- return -EINVAL; -++ if (!bpf_check_basics_ok(fprog->filter, fprog->len)) -++ return ERR_PTR(-EINVAL); -+ -+- prog = bpf_prog_alloc(bpf_fsize, 0); -++ prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0); -+ if (!prog) -+- return -ENOMEM; -++ return ERR_PTR(-ENOMEM); -+ -+ if (copy_from_user(prog->insns, fprog->filter, fsize)) { -+ __bpf_prog_free(prog); -+- return -EFAULT; -++ return ERR_PTR(-EINVAL); -+ } -+ -+ prog->len = fprog->len; -+@@ -1203,17 +1469,34 @@ int __sk_attach_filter(struct sock_fprog -+ err = bpf_prog_store_orig_filter(prog, fprog); -+ if (err) { -+ __bpf_prog_free(prog); -+- return -ENOMEM; -++ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* bpf_prepare_filter() already takes care of freeing -+ * memory in case something goes wrong. -+ */ -+- prog = bpf_prepare_filter(prog, NULL); -++ return bpf_prepare_filter(prog, NULL); -++} -++ -++/** -++ * sk_attach_filter - attach a socket filter -++ * @fprog: the filter program -++ * @sk: the socket to use -++ * -++ * Attach the user's filter code. We first run some sanity checks on -++ * it to make sure it does not explode on us later. If an error -++ * occurs or there is insufficient memory for the filter a negative -++ * errno code is returned. On success the return is zero. -++ */ -++int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) -++{ -++ struct bpf_prog *prog = __get_filter(fprog, sk); -++ int err; -++ -+ if (IS_ERR(prog)) -+ return PTR_ERR(prog); -+ -+- err = __sk_attach_prog(prog, sk, locked); -++ err = __sk_attach_prog(prog, sk); -+ if (err < 0) { -+ __bpf_prog_release(prog); -+ return err; -+@@ -1221,31 +1504,25 @@ int __sk_attach_filter(struct sock_fprog -+ -+ return 0; -+ } -+-EXPORT_SYMBOL_GPL(__sk_attach_filter); -++EXPORT_SYMBOL_GPL(sk_attach_filter); -+ -+-int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk) -++static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk) -+ { -+- return __sk_attach_filter(fprog, sk, sock_owned_by_user(sk)); -++ if (sock_flag(sk, SOCK_FILTER_LOCKED)) -++ return ERR_PTR(-EPERM); -++ -++ return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER); -+ } -+ -+ int sk_attach_bpf(u32 ufd, struct sock *sk) -+ { -+- struct bpf_prog *prog; -++ struct bpf_prog *prog = __get_bpf(ufd, sk); -+ int err; -+ -+- if (sock_flag(sk, SOCK_FILTER_LOCKED)) -+- return -EPERM; -+- -+- prog = bpf_prog_get(ufd); -+ if (IS_ERR(prog)) -+ return PTR_ERR(prog); -+ -+- if (prog->type != BPF_PROG_TYPE_SOCKET_FILTER) { -+- bpf_prog_put(prog); -+- return -EINVAL; -+- } -+- -+- err = __sk_attach_prog(prog, sk, sock_owned_by_user(sk)); -++ err = __sk_attach_prog(prog, sk); -+ if (err < 0) { -+ bpf_prog_put(prog); -+ return err; -+@@ -1254,79 +1531,201 @@ int sk_attach_bpf(u32 ufd, struct sock * -+ return 0; -+ } -+ -+-#define BPF_RECOMPUTE_CSUM(flags) ((flags) & 1) -++struct bpf_scratchpad { -++ union { -++ __be32 diff[MAX_BPF_STACK / sizeof(__be32)]; -++ u8 buff[MAX_BPF_STACK]; -++ }; -++}; -++ -++static DEFINE_PER_CPU(struct bpf_scratchpad, bpf_sp); -++ -++static inline int __bpf_try_make_writable(struct sk_buff *skb, -++ unsigned int write_len) -++{ -++ return skb_ensure_writable(skb, write_len); -++} -++ -++static inline int bpf_try_make_writable(struct sk_buff *skb, -++ unsigned int write_len) -++{ -++ int err = __bpf_try_make_writable(skb, write_len); -++ -++ bpf_compute_data_pointers(skb); -++ return err; -++} -++ -++static int bpf_try_make_head_writable(struct sk_buff *skb) -++{ -++ return bpf_try_make_writable(skb, skb_headlen(skb)); -++} -+ -+-static u64 bpf_skb_store_bytes(u64 r1, u64 r2, u64 r3, u64 r4, u64 flags) -++static inline void bpf_push_mac_rcsum(struct sk_buff *skb) -++{ -++ if (skb_at_tc_ingress(skb)) -++ skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len); -++} -++ -++static inline void bpf_pull_mac_rcsum(struct sk_buff *skb) -++{ -++ if (skb_at_tc_ingress(skb)) -++ skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len); -++} -++ -++BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset, -++ const void *, from, u32, len, u64, flags) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- int offset = (int) r2; -+- void *from = (void *) (long) r3; -+- unsigned int len = (unsigned int) r4; -+- char buf[16]; -+ void *ptr; -+ -+- /* bpf verifier guarantees that: -+- * 'from' pointer points to bpf program stack -+- * 'len' bytes of it were initialized -+- * 'len' > 0 -+- * 'skb' is a valid pointer to 'struct sk_buff' -+- * -+- * so check for invalid 'offset' and too large 'len' -+- */ -+- if (unlikely((u32) offset > 0xffff || len > sizeof(buf))) -++ if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH))) -++ return -EINVAL; -++ if (unlikely(offset > 0xffff)) -+ return -EFAULT; -+- if (unlikely(skb_try_make_writable(skb, offset + len))) -++ if (unlikely(bpf_try_make_writable(skb, offset + len))) -+ return -EFAULT; -+ -+- ptr = skb_header_pointer(skb, offset, len, buf); -+- if (unlikely(!ptr)) -+- return -EFAULT; -+- -+- if (BPF_RECOMPUTE_CSUM(flags)) -+- skb_postpull_rcsum(skb, ptr, len); -++ ptr = skb->data + offset; -++ if (flags & BPF_F_RECOMPUTE_CSUM) -++ __skb_postpull_rcsum(skb, ptr, len, offset); -+ -+ memcpy(ptr, from, len); -+ -+- if (ptr == buf) -+- /* skb_store_bits cannot return -EFAULT here */ -+- skb_store_bits(skb, offset, ptr, len); -++ if (flags & BPF_F_RECOMPUTE_CSUM) -++ __skb_postpush_rcsum(skb, ptr, len, offset); -++ if (flags & BPF_F_INVALIDATE_HASH) -++ skb_clear_hash(skb); -+ -+- if (BPF_RECOMPUTE_CSUM(flags) && skb->ip_summed == CHECKSUM_COMPLETE) -+- skb->csum = csum_add(skb->csum, csum_partial(ptr, len, 0)); -+ return 0; -+ } -+ -+-const struct bpf_func_proto bpf_skb_store_bytes_proto = { -++static const struct bpf_func_proto bpf_skb_store_bytes_proto = { -+ .func = bpf_skb_store_bytes, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_PTR_TO_CTX, -+ .arg2_type = ARG_ANYTHING, -+- .arg3_type = ARG_PTR_TO_STACK, -+- .arg4_type = ARG_CONST_STACK_SIZE, -++ .arg3_type = ARG_PTR_TO_MEM, -++ .arg4_type = ARG_CONST_SIZE, -+ .arg5_type = ARG_ANYTHING, -+ }; -+ -+-#define BPF_HEADER_FIELD_SIZE(flags) ((flags) & 0x0f) -+-#define BPF_IS_PSEUDO_HEADER(flags) ((flags) & 0x10) -++BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, -++ void *, to, u32, len) -++{ -++ void *ptr; -++ -++ if (unlikely(offset > 0xffff)) -++ goto err_clear; -++ -++ ptr = skb_header_pointer(skb, offset, len, to); -++ if (unlikely(!ptr)) -++ goto err_clear; -++ if (ptr != to) -++ memcpy(to, ptr, len); -+ -+-static u64 bpf_l3_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) -++ return 0; -++err_clear: -++ memset(to, 0, len); -++ return -EFAULT; -++} -++ -++static const struct bpf_func_proto bpf_skb_load_bytes_proto = { -++ .func = bpf_skb_load_bytes, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg4_type = ARG_CONST_SIZE, -++}; -++ -++BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, -++ u32, offset, void *, to, u32, len, u32, start_header) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- int offset = (int) r2; -+- __sum16 sum, *ptr; -++ u8 *end = skb_tail_pointer(skb); -++ u8 *start, *ptr; -+ -+- if (unlikely((u32) offset > 0xffff)) -+- return -EFAULT; -++ if (unlikely(offset > 0xffff)) -++ goto err_clear; -+ -+- if (unlikely(skb_try_make_writable(skb, offset + sizeof(sum)))) -+- return -EFAULT; -++ switch (start_header) { -++ case BPF_HDR_START_MAC: -++ if (unlikely(!skb_mac_header_was_set(skb))) -++ goto err_clear; -++ start = skb_mac_header(skb); -++ break; -++ case BPF_HDR_START_NET: -++ start = skb_network_header(skb); -++ break; -++ default: -++ goto err_clear; -++ } -+ -+- ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); -+- if (unlikely(!ptr)) -++ ptr = start + offset; -++ -++ if (likely(ptr + len <= end)) { -++ memcpy(to, ptr, len); -++ return 0; -++ } -++ -++err_clear: -++ memset(to, 0, len); -++ return -EFAULT; -++} -++ -++static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = { -++ .func = bpf_skb_load_bytes_relative, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg4_type = ARG_CONST_SIZE, -++ .arg5_type = ARG_ANYTHING, -++}; -++ -++BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len) -++{ -++ /* Idea is the following: should the needed direct read/write -++ * test fail during runtime, we can pull in more data and redo -++ * again, since implicitly, we invalidate previous checks here. -++ * -++ * Or, since we know how much we need to make read/writeable, -++ * this can be done once at the program beginning for direct -++ * access case. By this we overcome limitations of only current -++ * headroom being accessible. -++ */ -++ return bpf_try_make_writable(skb, len ? : skb_headlen(skb)); -++} -++ -++static const struct bpf_func_proto bpf_skb_pull_data_proto = { -++ .func = bpf_skb_pull_data, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++}; -++ -++BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset, -++ u64, from, u64, to, u64, flags) -++{ -++ __sum16 *ptr; -++ -++ if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK))) -++ return -EINVAL; -++ if (unlikely(offset > 0xffff || offset & 1)) -++ return -EFAULT; -++ if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) -+ return -EFAULT; -+ -+- switch (BPF_HEADER_FIELD_SIZE(flags)) { -++ ptr = (__sum16 *)(skb->data + offset); -++ switch (flags & BPF_F_HDR_FIELD_MASK) { -++ case 0: -++ if (unlikely(from != 0)) -++ return -EINVAL; -++ -++ csum_replace_by_diff(ptr, to); -++ break; -+ case 2: -+ csum_replace2(ptr, from, to); -+ break; -+@@ -1337,14 +1736,10 @@ static u64 bpf_l3_csum_replace(u64 r1, u -+ return -EINVAL; -+ } -+ -+- if (ptr == &sum) -+- /* skb_store_bits guaranteed to not return -EFAULT here */ -+- skb_store_bits(skb, offset, ptr, sizeof(sum)); -+- -+ return 0; -+ } -+ -+-const struct bpf_func_proto bpf_l3_csum_replace_proto = { -++static const struct bpf_func_proto bpf_l3_csum_replace_proto = { -+ .func = bpf_l3_csum_replace, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+@@ -1355,23 +1750,33 @@ const struct bpf_func_proto bpf_l3_csum_ -+ .arg5_type = ARG_ANYTHING, -+ }; -+ -+-static u64 bpf_l4_csum_replace(u64 r1, u64 r2, u64 from, u64 to, u64 flags) -++BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset, -++ u64, from, u64, to, u64, flags) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- bool is_pseudo = !!BPF_IS_PSEUDO_HEADER(flags); -+- int offset = (int) r2; -+- __sum16 sum, *ptr; -++ bool is_pseudo = flags & BPF_F_PSEUDO_HDR; -++ bool is_mmzero = flags & BPF_F_MARK_MANGLED_0; -++ bool do_mforce = flags & BPF_F_MARK_ENFORCE; -++ __sum16 *ptr; -+ -+- if (unlikely((u32) offset > 0xffff)) -++ if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE | -++ BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK))) -++ return -EINVAL; -++ if (unlikely(offset > 0xffff || offset & 1)) -+ return -EFAULT; -+- if (unlikely(skb_try_make_writable(skb, offset + sizeof(sum)))) -++ if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr)))) -+ return -EFAULT; -+ -+- ptr = skb_header_pointer(skb, offset, sizeof(sum), &sum); -+- if (unlikely(!ptr)) -+- return -EFAULT; -++ ptr = (__sum16 *)(skb->data + offset); -++ if (is_mmzero && !do_mforce && !*ptr) -++ return 0; -+ -+- switch (BPF_HEADER_FIELD_SIZE(flags)) { -++ switch (flags & BPF_F_HDR_FIELD_MASK) { -++ case 0: -++ if (unlikely(from != 0)) -++ return -EINVAL; -++ -++ inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo); -++ break; -+ case 2: -+ inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo); -+ break; -+@@ -1382,14 +1787,12 @@ static u64 bpf_l4_csum_replace(u64 r1, u -+ return -EINVAL; -+ } -+ -+- if (ptr == &sum) -+- /* skb_store_bits guaranteed to not return -EFAULT here */ -+- skb_store_bits(skb, offset, ptr, sizeof(sum)); -+- -++ if (is_mmzero && !*ptr) -++ *ptr = CSUM_MANGLED_0; -+ return 0; -+ } -+ -+-const struct bpf_func_proto bpf_l4_csum_replace_proto = { -++static const struct bpf_func_proto bpf_l4_csum_replace_proto = { -+ .func = bpf_l4_csum_replace, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+@@ -1400,30 +1803,172 @@ const struct bpf_func_proto bpf_l4_csum_ -+ .arg5_type = ARG_ANYTHING, -+ }; -+ -+-#define BPF_IS_REDIRECT_INGRESS(flags) ((flags) & 1) -++BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size, -++ __be32 *, to, u32, to_size, __wsum, seed) -++{ -++ struct bpf_scratchpad *sp = this_cpu_ptr(&bpf_sp); -++ u32 diff_size = from_size + to_size; -++ int i, j = 0; -++ -++ /* This is quite flexible, some examples: -++ * -++ * from_size == 0, to_size > 0, seed := csum --> pushing data -++ * from_size > 0, to_size == 0, seed := csum --> pulling data -++ * from_size > 0, to_size > 0, seed := 0 --> diffing data -++ * -++ * Even for diffing, from_size and to_size don't need to be equal. -++ */ -++ if (unlikely(((from_size | to_size) & (sizeof(__be32) - 1)) || -++ diff_size > sizeof(sp->diff))) -++ return -EINVAL; -++ -++ for (i = 0; i < from_size / sizeof(__be32); i++, j++) -++ sp->diff[j] = ~from[i]; -++ for (i = 0; i < to_size / sizeof(__be32); i++, j++) -++ sp->diff[j] = to[i]; -++ -++ return csum_partial(sp->diff, diff_size, seed); -++} -++ -++static const struct bpf_func_proto bpf_csum_diff_proto = { -++ .func = bpf_csum_diff, -++ .gpl_only = false, -++ .pkt_access = true, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_MEM_OR_NULL, -++ .arg2_type = ARG_CONST_SIZE_OR_ZERO, -++ .arg3_type = ARG_PTR_TO_MEM_OR_NULL, -++ .arg4_type = ARG_CONST_SIZE_OR_ZERO, -++ .arg5_type = ARG_ANYTHING, -++}; -++ -++BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum) -++{ -++ /* The interface is to be used in combination with bpf_csum_diff() -++ * for direct packet writes. csum rotation for alignment as well -++ * as emulating csum_sub() can be done from the eBPF program. -++ */ -++ if (skb->ip_summed == CHECKSUM_COMPLETE) -++ return (skb->csum = csum_add(skb->csum, csum)); -++ -++ return -ENOTSUPP; -++} -++ -++static const struct bpf_func_proto bpf_csum_update_proto = { -++ .func = bpf_csum_update, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++}; -++ -++static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb) -++{ -++ return dev_forward_skb(dev, skb); -++} -++ -++static inline int __bpf_rx_skb_no_mac(struct net_device *dev, -++ struct sk_buff *skb) -++{ -++ int ret = ____dev_forward_skb(dev, skb); -++ -++ if (likely(!ret)) { -++ skb->dev = dev; -++ ret = netif_rx(skb); -++ } -++ -++ return ret; -++} -++ -++static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb) -++{ -++ int ret; -++ -++ skb->dev = dev; -++ skb->tstamp.tv64 = 0; -++ -++ ret = dev_queue_xmit(skb); -++ -++ return ret; -++} -++ -++static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev, -++ u32 flags) -++{ -++ unsigned int mlen = skb_network_offset(skb); -++ -++ if (mlen) { -++ __skb_pull(skb, mlen); -++ -++ /* At ingress, the mac header has already been pulled once. -++ * At egress, skb_pospull_rcsum has to be done in case that -++ * the skb is originated from ingress (i.e. a forwarded skb) -++ * to ensure that rcsum starts at net header. -++ */ -++ if (!skb_at_tc_ingress(skb)) -++ skb_postpull_rcsum(skb, skb_mac_header(skb), mlen); -++ } -++ skb_pop_mac_header(skb); -++ skb_reset_mac_len(skb); -++ return flags & BPF_F_INGRESS ? -++ __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb); -++} -+ -+-static u64 bpf_clone_redirect(u64 r1, u64 ifindex, u64 flags, u64 r4, u64 r5) -++static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev, -++ u32 flags) -++{ -++ /* Verify that a link layer header is carried */ -++ if (unlikely(skb->mac_header >= skb->network_header)) { -++ kfree_skb(skb); -++ return -ERANGE; -++ } -++ -++ bpf_push_mac_rcsum(skb); -++ return flags & BPF_F_INGRESS ? -++ __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb); -++} -++ -++static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev, -++ u32 flags) -++{ -++ if (dev_is_mac_header_xmit(dev)) -++ return __bpf_redirect_common(skb, dev, flags); -++ else -++ return __bpf_redirect_no_mac(skb, dev, flags); -++} -++ -++BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1, *skb2; -+ struct net_device *dev; -++ struct sk_buff *clone; -++ int ret; -++ -++ if (unlikely(flags & ~(BPF_F_INGRESS))) -++ return -EINVAL; -+ -+ dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex); -+ if (unlikely(!dev)) -+ return -EINVAL; -+ -+- skb2 = skb_clone(skb, GFP_ATOMIC); -+- if (unlikely(!skb2)) -++ clone = skb_clone(skb, GFP_ATOMIC); -++ if (unlikely(!clone)) -+ return -ENOMEM; -+ -+- if (BPF_IS_REDIRECT_INGRESS(flags)) -+- return dev_forward_skb(dev, skb2); -++ /* For direct write, we need to keep the invariant that the skbs -++ * we're dealing with need to be uncloned. Should uncloning fail -++ * here, we need to free the just generated clone to unclone once -++ * again. -++ */ -++ ret = bpf_try_make_head_writable(skb); -++ if (unlikely(ret)) { -++ kfree_skb(clone); -++ return -ENOMEM; -++ } -+ -+- skb2->dev = dev; -+- skb_sender_cpu_clear(skb2); -+- return dev_queue_xmit(skb2); -++ return __bpf_redirect(clone, dev, flags); -+ } -+ -+-const struct bpf_func_proto bpf_clone_redirect_proto = { -++static const struct bpf_func_proto bpf_clone_redirect_proto = { -+ .func = bpf_clone_redirect, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+@@ -1432,42 +1977,38 @@ const struct bpf_func_proto bpf_clone_re -+ .arg3_type = ARG_ANYTHING, -+ }; -+ -+-struct redirect_info { -+- u32 ifindex; -+- u32 flags; -+-}; -++DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info); -++EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); -+ -+-static DEFINE_PER_CPU(struct redirect_info, redirect_info); -+-static u64 bpf_redirect(u64 ifindex, u64 flags, u64 r3, u64 r4, u64 r5) -++BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) -+ { -+- struct redirect_info *ri = this_cpu_ptr(&redirect_info); -++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); -++ -++ if (unlikely(flags & ~(BPF_F_INGRESS))) -++ return TC_ACT_SHOT; -+ -+- ri->ifindex = ifindex; -+ ri->flags = flags; -++ ri->tgt_index = ifindex; -++ -+ return TC_ACT_REDIRECT; -+ } -+ -+ int skb_do_redirect(struct sk_buff *skb) -+ { -+- struct redirect_info *ri = this_cpu_ptr(&redirect_info); -++ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info); -+ struct net_device *dev; -+ -+- dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->ifindex); -+- ri->ifindex = 0; -++ dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index); -++ ri->tgt_index = 0; -+ if (unlikely(!dev)) { -+ kfree_skb(skb); -+ return -EINVAL; -+ } -+ -+- if (BPF_IS_REDIRECT_INGRESS(ri->flags)) -+- return dev_forward_skb(dev, skb); -+- -+- skb->dev = dev; -+- skb_sender_cpu_clear(skb); -+- return dev_queue_xmit(skb); -++ return __bpf_redirect(skb, dev, ri->flags); -+ } -+ -+-const struct bpf_func_proto bpf_redirect_proto = { -++static const struct bpf_func_proto bpf_redirect_proto = { -+ .func = bpf_redirect, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+@@ -1475,50 +2016,75 @@ const struct bpf_func_proto bpf_redirect -+ .arg2_type = ARG_ANYTHING, -+ }; -+ -+-static u64 bpf_get_cgroup_classid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb) -+ { -+- return task_get_classid((struct sk_buff *) (unsigned long) r1); -++ /* If skb_clear_hash() was called due to mangling, we can -++ * trigger SW recalculation here. Later access to hash -++ * can then use the inline skb->hash via context directly -++ * instead of calling this helper again. -++ */ -++ return skb_get_hash(skb); -+ } -+ -+-static const struct bpf_func_proto bpf_get_cgroup_classid_proto = { -+- .func = bpf_get_cgroup_classid, -+- .gpl_only = false, -+- .ret_type = RET_INTEGER, -+- .arg1_type = ARG_PTR_TO_CTX, -++static const struct bpf_func_proto bpf_get_hash_recalc_proto = { -++ .func = bpf_get_hash_recalc, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -+ }; -+ -+-static u64 bpf_get_route_realm(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb) -+ { -+-#ifdef CONFIG_IP_ROUTE_CLASSID -+- const struct dst_entry *dst; -++ /* After all direct packet write, this can be used once for -++ * triggering a lazy recalc on next skb_get_hash() invocation. -++ */ -++ skb_clear_hash(skb); -++ return 0; -++} -+ -+- dst = skb_dst((struct sk_buff *) (unsigned long) r1); -+- if (dst) -+- return dst->tclassid; -+-#endif -++static const struct bpf_func_proto bpf_set_hash_invalid_proto = { -++ .func = bpf_set_hash_invalid, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++}; -++ -++BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash) -++{ -++ /* Set user specified hash as L4(+), so that it gets returned -++ * on skb_get_hash() call unless BPF prog later on triggers a -++ * skb_clear_hash(). -++ */ -++ __skb_set_sw_hash(skb, hash, true); -+ return 0; -+ } -+ -+-static const struct bpf_func_proto bpf_get_route_realm_proto = { -+- .func = bpf_get_route_realm, -+- .gpl_only = false, -+- .ret_type = RET_INTEGER, -+- .arg1_type = ARG_PTR_TO_CTX, -++static const struct bpf_func_proto bpf_set_hash_proto = { -++ .func = bpf_set_hash, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -+ }; -+ -+-static u64 bpf_skb_vlan_push(u64 r1, u64 r2, u64 vlan_tci, u64 r4, u64 r5) -++BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto, -++ u16, vlan_tci) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- __be16 vlan_proto = (__force __be16) r2; -++ int ret; -+ -+ if (unlikely(vlan_proto != htons(ETH_P_8021Q) && -+ vlan_proto != htons(ETH_P_8021AD))) -+ vlan_proto = htons(ETH_P_8021Q); -+ -+- return skb_vlan_push(skb, vlan_proto, vlan_tci); -++ bpf_push_mac_rcsum(skb); -++ ret = skb_vlan_push(skb, vlan_proto, vlan_tci); -++ bpf_pull_mac_rcsum(skb); -++ -++ bpf_compute_data_pointers(skb); -++ return ret; -+ } -+ -+-const struct bpf_func_proto bpf_skb_vlan_push_proto = { -++static const struct bpf_func_proto bpf_skb_vlan_push_proto = { -+ .func = bpf_skb_vlan_push, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+@@ -1526,116 +2092,401 @@ const struct bpf_func_proto bpf_skb_vlan -+ .arg2_type = ARG_ANYTHING, -+ .arg3_type = ARG_ANYTHING, -+ }; -+-EXPORT_SYMBOL_GPL(bpf_skb_vlan_push_proto); -+ -+-static u64 bpf_skb_vlan_pop(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) -++BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -++ int ret; -++ -++ bpf_push_mac_rcsum(skb); -++ ret = skb_vlan_pop(skb); -++ bpf_pull_mac_rcsum(skb); -+ -+- return skb_vlan_pop(skb); -++ bpf_compute_data_pointers(skb); -++ return ret; -+ } -+ -+-const struct bpf_func_proto bpf_skb_vlan_pop_proto = { -++static const struct bpf_func_proto bpf_skb_vlan_pop_proto = { -+ .func = bpf_skb_vlan_pop, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_PTR_TO_CTX, -+ }; -+-EXPORT_SYMBOL_GPL(bpf_skb_vlan_pop_proto); -+ -+-bool bpf_helper_changes_skb_data(void *func) -++BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type) -+ { -+- if (func == bpf_skb_vlan_push) -+- return true; -+- if (func == bpf_skb_vlan_pop) -+- return true; -+- if (func == bpf_skb_store_bytes) -+- return true; -+- if (func == bpf_l3_csum_replace) -+- return true; -+- if (func == bpf_l4_csum_replace) -+- return true; -++ /* We only allow a restricted subset to be changed for now. */ -++ if (unlikely(!skb_pkt_type_ok(skb->pkt_type) || -++ !skb_pkt_type_ok(pkt_type))) -++ return -EINVAL; -+ -+- return false; -++ skb->pkt_type = pkt_type; -++ return 0; -++} -++ -++static const struct bpf_func_proto bpf_skb_change_type_proto = { -++ .func = bpf_skb_change_type, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++}; -++ -++#define BPF_F_ADJ_ROOM_ENCAP_L3_MASK (BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \ -++ BPF_F_ADJ_ROOM_ENCAP_L3_IPV6) -++ -++#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \ -++ BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \ -++ BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \ -++ BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \ -++ BPF_F_ADJ_ROOM_ENCAP_L2( \ -++ BPF_ADJ_ROOM_ENCAP_L2_MASK)) -++ -++#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC -++ -++static u32 __bpf_skb_min_len(const struct sk_buff *skb) -++{ -++ u32 min_len = skb_network_offset(skb); -++ -++ if (skb_transport_header_was_set(skb)) -++ min_len = skb_transport_offset(skb); -++ if (skb->ip_summed == CHECKSUM_PARTIAL) -++ min_len = skb_checksum_start_offset(skb) + -++ skb->csum_offset + sizeof(__sum16); -++ return min_len; -+ } -+ -+-static u64 bpf_skb_get_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) -++static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- struct bpf_tunnel_key *to = (struct bpf_tunnel_key *) (long) r2; -+- struct ip_tunnel_info *info = skb_tunnel_info(skb); -++ unsigned int old_len = skb->len; -++ int ret; -+ -+- if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags || !info)) -++ ret = __skb_grow_rcsum(skb, new_len); -++ if (!ret) -++ memset(skb->data + old_len, 0, new_len - old_len); -++ return ret; -++} -++ -++static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) -++{ -++ return __skb_trim_rcsum(skb, new_len); -++} -++ -++static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len, -++ u64 flags) -++{ -++ u32 max_len = BPF_SKB_MAX_LEN; -++ u32 min_len = __bpf_skb_min_len(skb); -++ int ret; -++ -++ if (unlikely(flags || new_len > max_len || new_len < min_len)) -+ return -EINVAL; -+- if (ip_tunnel_info_af(info) != AF_INET) -++ if (skb->encapsulation) -++ return -ENOTSUPP; -++ -++ /* The basic idea of this helper is that it's performing the -++ * needed work to either grow or trim an skb, and eBPF program -++ * rewrites the rest via helpers like bpf_skb_store_bytes(), -++ * bpf_lX_csum_replace() and others rather than passing a raw -++ * buffer here. This one is a slow path helper and intended -++ * for replies with control messages. -++ * -++ * Like in bpf_skb_change_proto(), we want to keep this rather -++ * minimal and without protocol specifics so that we are able -++ * to separate concerns as in bpf_skb_store_bytes() should only -++ * be the one responsible for writing buffers. -++ * -++ * It's really expected to be a slow path operation here for -++ * control message replies, so we're implicitly linearizing, -++ * uncloning and drop offloads from the skb by this. -++ */ -++ ret = __bpf_try_make_writable(skb, skb->len); -++ if (!ret) { -++ if (new_len > skb->len) -++ ret = bpf_skb_grow_rcsum(skb, new_len); -++ else if (new_len < skb->len) -++ ret = bpf_skb_trim_rcsum(skb, new_len); -++ if (!ret && skb_is_gso(skb)) -++ skb_gso_reset(skb); -++ } -++ return ret; -++} -++ -++BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len, -++ u64, flags) -++{ -++ int ret = __bpf_skb_change_tail(skb, new_len, flags); -++ -++ bpf_compute_data_pointers(skb); -++ return ret; -++} -++ -++static const struct bpf_func_proto bpf_skb_change_tail_proto = { -++ .func = bpf_skb_change_tail, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_ANYTHING, -++}; -++ -++static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room, -++ u64 flags) -++{ -++ u32 max_len = BPF_SKB_MAX_LEN; -++ u32 new_len = skb->len + head_room; -++ int ret; -++ -++ if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) || -++ new_len < skb->len)) -+ return -EINVAL; -+ -+- to->tunnel_id = be64_to_cpu(info->key.tun_id); -+- to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src); -++ ret = skb_cow(skb, head_room); -++ if (likely(!ret)) { -++ /* Idea for this helper is that we currently only -++ * allow to expand on mac header. This means that -++ * skb->protocol network header, etc, stay as is. -++ * Compared to bpf_skb_change_tail(), we're more -++ * flexible due to not needing to linearize or -++ * reset GSO. Intention for this helper is to be -++ * used by an L3 skb that needs to push mac header -++ * for redirection into L2 device. -++ */ -++ __skb_push(skb, head_room); -++ memset(skb->data, 0, head_room); -++ skb_reset_mac_header(skb); -++ skb_reset_mac_len(skb); -++ } -+ -+- return 0; -++ return ret; -+ } -+ -+-const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = { -+- .func = bpf_skb_get_tunnel_key, -++BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room, -++ u64, flags) -++{ -++ int ret = __bpf_skb_change_head(skb, head_room, flags); -++ -++ bpf_compute_data_pointers(skb); -++ return ret; -++} -++ -++static const struct bpf_func_proto bpf_skb_change_head_proto = { -++ .func = bpf_skb_change_head, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_PTR_TO_CTX, -+- .arg2_type = ARG_PTR_TO_STACK, -+- .arg3_type = ARG_CONST_STACK_SIZE, -+- .arg4_type = ARG_ANYTHING, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_ANYTHING, -+ }; -+ -+-static struct metadata_dst __percpu *md_dst; -++void bpf_clear_redirect_map(struct bpf_map *map) -++{ -++ struct bpf_redirect_info *ri; -++ int cpu; -++ -++ for_each_possible_cpu(cpu) { -++ ri = per_cpu_ptr(&bpf_redirect_info, cpu); -++ /* Avoid polluting remote cacheline due to writes if -++ * not needed. Once we pass this test, we need the -++ * cmpxchg() to make sure it hasn't been changed in -++ * the meantime by remote CPU. -++ */ -++ if (unlikely(READ_ONCE(ri->map) == map)) -++ cmpxchg(&ri->map, map, NULL); -++ } -++} -++ -++static unsigned long bpf_skb_copy(void *dst_buff, const void *skb, -++ unsigned long off, unsigned long len) -++{ -++ void *ptr = skb_header_pointer(skb, off, len, dst_buff); -++ -++ if (unlikely(!ptr)) -++ return len; -++ if (ptr != dst_buff) -++ memcpy(dst_buff, ptr, len); -++ -++ return 0; -++} -+ -+-static u64 bpf_skb_set_tunnel_key(u64 r1, u64 r2, u64 size, u64 flags, u64 r5) -++BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map, -++ u64, flags, void *, meta, u64, meta_size) -+ { -+- struct sk_buff *skb = (struct sk_buff *) (long) r1; -+- struct bpf_tunnel_key *from = (struct bpf_tunnel_key *) (long) r2; -+- struct metadata_dst *md = this_cpu_ptr(md_dst); -+- struct ip_tunnel_info *info; -++ u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32; -+ -+- if (unlikely(size != sizeof(struct bpf_tunnel_key) || flags)) -++ if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK))) -+ return -EINVAL; -++ if (unlikely(skb_size > skb->len)) -++ return -EFAULT; -+ -+- skb_dst_drop(skb); -+- dst_hold((struct dst_entry *) md); -+- skb_dst_set(skb, (struct dst_entry *) md); -++ return bpf_event_output(map, flags, meta, meta_size, skb, skb_size, -++ bpf_skb_copy); -++} -+ -+- info = &md->u.tun_info; -+- info->mode = IP_TUNNEL_INFO_TX; -+- info->key.tun_flags = TUNNEL_KEY; -+- info->key.tun_id = cpu_to_be64(from->tunnel_id); -+- info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4); -++static const struct bpf_func_proto bpf_skb_event_output_proto = { -++ .func = bpf_skb_event_output, -++ .gpl_only = true, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_CONST_MAP_PTR, -++ .arg3_type = ARG_ANYTHING, -++ .arg4_type = ARG_PTR_TO_MEM, -++ .arg5_type = ARG_CONST_SIZE_OR_ZERO, -++}; -++ -++ -++const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; -++EXPORT_SYMBOL_GPL(ipv6_bpf_stub); -++ -++#ifdef CONFIG_XFRM -++BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index, -++ struct bpf_xfrm_state *, to, u32, size, u64, flags) -++{ -++ const struct sec_path *sp = skb_sec_path(skb); -++ const struct xfrm_state *x; -++ -++ if (!sp || unlikely(index >= sp->len || flags)) -++ goto err_clear; -++ -++ x = sp->xvec[index]; -++ -++ if (unlikely(size != sizeof(struct bpf_xfrm_state))) -++ goto err_clear; -++ -++ to->reqid = x->props.reqid; -++ to->spi = x->id.spi; -++ to->family = x->props.family; -++ to->ext = 0; -++ -++ if (to->family == AF_INET6) { -++ memcpy(to->remote_ipv6, x->props.saddr.a6, -++ sizeof(to->remote_ipv6)); -++ } else { -++ to->remote_ipv4 = x->props.saddr.a4; -++ memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3); -++ } -+ -+ return 0; -++err_clear: -++ memset(to, 0, size); -++ return -EINVAL; -+ } -+ -+-const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { -+- .func = bpf_skb_set_tunnel_key, -++static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { -++ .func = bpf_skb_get_xfrm_state, -+ .gpl_only = false, -+ .ret_type = RET_INTEGER, -+ .arg1_type = ARG_PTR_TO_CTX, -+- .arg2_type = ARG_PTR_TO_STACK, -+- .arg3_type = ARG_CONST_STACK_SIZE, -+- .arg4_type = ARG_ANYTHING, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_PTR_TO_UNINIT_MEM, -++ .arg4_type = ARG_CONST_SIZE, -++ .arg5_type = ARG_ANYTHING, -+ }; -++#endif -++ -+ -+-static const struct bpf_func_proto *bpf_get_skb_set_tunnel_key_proto(void) -++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) -++static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, -++ bool ingress) -+ { -+- if (!md_dst) { -+- /* race is not possible, since it's called from -+- * verifier that is holding verifier mutex -+- */ -+- md_dst = metadata_dst_alloc_percpu(0, GFP_KERNEL); -+- if (!md_dst) -+- return NULL; -++ return bpf_lwt_push_ip_encap(skb, hdr, len, ingress); -++} -++#endif -++ -++BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr, -++ u32, len) -++{ -++ switch (type) { -++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) -++ case BPF_LWT_ENCAP_IP: -++ return bpf_push_ip_encap(skb, hdr, len, true /* ingress */); -++#endif -++ default: -++ return -EINVAL; -+ } -+- return &bpf_skb_set_tunnel_key_proto; -++} -++ -++BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type, -++ void *, hdr, u32, len) -++{ -++ switch (type) { -++#if IS_ENABLED(CONFIG_LWTUNNEL_BPF) -++ case BPF_LWT_ENCAP_IP: -++ return bpf_push_ip_encap(skb, hdr, len, false /* egress */); -++#endif -++ default: -++ return -EINVAL; -++ } -++} -++ -++static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { -++ .func = bpf_lwt_in_push_encap, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_PTR_TO_MEM, -++ .arg4_type = ARG_CONST_SIZE -++}; -++ -++static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { -++ .func = bpf_lwt_xmit_push_encap, -++ .gpl_only = false, -++ .ret_type = RET_INTEGER, -++ .arg1_type = ARG_PTR_TO_CTX, -++ .arg2_type = ARG_ANYTHING, -++ .arg3_type = ARG_PTR_TO_MEM, -++ .arg4_type = ARG_CONST_SIZE -++}; -++ -++bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++BPF_CALL_1(bpf_tcp_sock, struct sock *, sk) -++{ -++ if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP) -++ return (unsigned long)sk; -++ -++ return (unsigned long)NULL; -++} -++ -++const struct bpf_func_proto bpf_tcp_sock_proto = { -++ .func = bpf_tcp_sock, -++ .gpl_only = false, -++ .ret_type = RET_PTR_TO_TCP_SOCK_OR_NULL, -++ .arg1_type = ARG_PTR_TO_SOCK_COMMON, -++}; -++ -++bool bpf_helper_changes_pkt_data(void *func) -++{ -++ if (func == bpf_skb_vlan_push || -++ func == bpf_skb_vlan_pop || -++ func == bpf_skb_store_bytes || -++ func == bpf_skb_change_head || -++ func == bpf_skb_change_tail || -++ func == bpf_skb_pull_data || -++ func == bpf_clone_redirect || -++ func == bpf_l3_csum_replace || -++ func == bpf_l4_csum_replace || -++ func == bpf_lwt_in_push_encap || -++ func == bpf_lwt_xmit_push_encap) -++ return true; -++ -++ return false; -+ } -+ -+ static const struct bpf_func_proto * -+-sk_filter_func_proto(enum bpf_func_id func_id) -++bpf_base_func_proto(enum bpf_func_id func_id) -+ { -+ switch (func_id) { -+ case BPF_FUNC_map_lookup_elem: -+@@ -1644,283 +2495,1168 @@ sk_filter_func_proto(enum bpf_func_id fu -+ return &bpf_map_update_elem_proto; -+ case BPF_FUNC_map_delete_elem: -+ return &bpf_map_delete_elem_proto; -++ case BPF_FUNC_map_push_elem: -++ return &bpf_map_push_elem_proto; -++ case BPF_FUNC_map_pop_elem: -++ return &bpf_map_pop_elem_proto; -++ case BPF_FUNC_map_peek_elem: -++ return &bpf_map_peek_elem_proto; -+ case BPF_FUNC_get_prandom_u32: -+ return &bpf_get_prandom_u32_proto; -+ case BPF_FUNC_get_smp_processor_id: -+- return &bpf_get_smp_processor_id_proto; -++ return &bpf_get_raw_smp_processor_id_proto; -++ case BPF_FUNC_get_numa_node_id: -++ return &bpf_get_numa_node_id_proto; -+ case BPF_FUNC_tail_call: -+ return &bpf_tail_call_proto; -+ case BPF_FUNC_ktime_get_ns: -+ return &bpf_ktime_get_ns_proto; -++ default: -++ break; -++ } -++ -++ if (!capable(CAP_SYS_ADMIN)) -++ return NULL; -++ -++ switch (func_id) { -++ case BPF_FUNC_spin_lock: -++ return &bpf_spin_lock_proto; -++ case BPF_FUNC_spin_unlock: -++ return &bpf_spin_unlock_proto; -+ case BPF_FUNC_trace_printk: -+- if (capable(CAP_SYS_ADMIN)) -+- return bpf_get_trace_printk_proto(); -++ return bpf_get_trace_printk_proto(); -+ default: -+ return NULL; -+ } -+ } -+ -+ static const struct bpf_func_proto * -+-tc_cls_act_func_proto(enum bpf_func_id func_id) -++sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ return bpf_base_func_proto(func_id); -++} -++ -++static const struct bpf_func_proto * -++sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ return bpf_base_func_proto(func_id); -++} -++ -++const struct bpf_func_proto bpf_sk_storage_get_proto __weak; -++const struct bpf_func_proto bpf_sk_storage_delete_proto __weak; -++ -++static const struct bpf_func_proto * -++tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -+ { -+ switch (func_id) { -+ case BPF_FUNC_skb_store_bytes: -+ return &bpf_skb_store_bytes_proto; -++ case BPF_FUNC_skb_load_bytes: -++ return &bpf_skb_load_bytes_proto; -++ case BPF_FUNC_skb_load_bytes_relative: -++ return &bpf_skb_load_bytes_relative_proto; -++ case BPF_FUNC_skb_pull_data: -++ return &bpf_skb_pull_data_proto; -++ case BPF_FUNC_csum_diff: -++ return &bpf_csum_diff_proto; -++ case BPF_FUNC_csum_update: -++ return &bpf_csum_update_proto; -+ case BPF_FUNC_l3_csum_replace: -+ return &bpf_l3_csum_replace_proto; -+ case BPF_FUNC_l4_csum_replace: -+ return &bpf_l4_csum_replace_proto; -+ case BPF_FUNC_clone_redirect: -+ return &bpf_clone_redirect_proto; -+- case BPF_FUNC_get_cgroup_classid: -+- return &bpf_get_cgroup_classid_proto; -+ case BPF_FUNC_skb_vlan_push: -+ return &bpf_skb_vlan_push_proto; -+ case BPF_FUNC_skb_vlan_pop: -+ return &bpf_skb_vlan_pop_proto; -+- case BPF_FUNC_skb_get_tunnel_key: -+- return &bpf_skb_get_tunnel_key_proto; -+- case BPF_FUNC_skb_set_tunnel_key: -+- return bpf_get_skb_set_tunnel_key_proto(); -++ case BPF_FUNC_skb_change_type: -++ return &bpf_skb_change_type_proto; -++ case BPF_FUNC_skb_change_tail: -++ return &bpf_skb_change_tail_proto; -++ case BPF_FUNC_redirect: -++ return &bpf_redirect_proto; -++ case BPF_FUNC_get_hash_recalc: -++ return &bpf_get_hash_recalc_proto; -++ case BPF_FUNC_set_hash_invalid: -++ return &bpf_set_hash_invalid_proto; -++ case BPF_FUNC_set_hash: -++ return &bpf_set_hash_proto; -++ case BPF_FUNC_perf_event_output: -++ return &bpf_skb_event_output_proto; -++ case BPF_FUNC_get_smp_processor_id: -++ return &bpf_get_smp_processor_id_proto; -++#ifdef CONFIG_XFRM -++ case BPF_FUNC_skb_get_xfrm_state: -++ return &bpf_skb_get_xfrm_state_proto; -++#endif -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++static const struct bpf_func_proto * -++xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++const struct bpf_func_proto bpf_sock_map_update_proto __weak; -++const struct bpf_func_proto bpf_sock_hash_update_proto __weak; -++ -++static const struct bpf_func_proto * -++sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++const struct bpf_func_proto bpf_msg_redirect_map_proto __weak; -++const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak; -++ -++static const struct bpf_func_proto * -++sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++const struct bpf_func_proto bpf_sk_redirect_map_proto __weak; -++const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak; -++ -++static const struct bpf_func_proto * -++sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ return bpf_base_func_proto(func_id); -++} -++ -++static const struct bpf_func_proto * -++flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++static const struct bpf_func_proto * -++lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ case BPF_FUNC_skb_load_bytes: -++ return &bpf_skb_load_bytes_proto; -++ case BPF_FUNC_skb_pull_data: -++ return &bpf_skb_pull_data_proto; -++ case BPF_FUNC_csum_diff: -++ return &bpf_csum_diff_proto; -++ case BPF_FUNC_get_hash_recalc: -++ return &bpf_get_hash_recalc_proto; -++ case BPF_FUNC_perf_event_output: -++ return &bpf_skb_event_output_proto; -++ case BPF_FUNC_get_smp_processor_id: -++ return &bpf_get_smp_processor_id_proto; -++ default: -++ return bpf_base_func_proto(func_id); -++ } -++} -++ -++static const struct bpf_func_proto * -++lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -++ case BPF_FUNC_lwt_push_encap: -++ return &bpf_lwt_in_push_encap_proto; -++ default: -++ return lwt_out_func_proto(func_id, prog); -++ } -++} -++ -++static const struct bpf_func_proto * -++lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -++{ -++ switch (func_id) { -+ case BPF_FUNC_redirect: -+ return &bpf_redirect_proto; -+- case BPF_FUNC_get_route_realm: -+- return &bpf_get_route_realm_proto; -++ case BPF_FUNC_clone_redirect: -++ return &bpf_clone_redirect_proto; -++ case BPF_FUNC_skb_change_tail: -++ return &bpf_skb_change_tail_proto; -++ case BPF_FUNC_skb_change_head: -++ return &bpf_skb_change_head_proto; -++ case BPF_FUNC_skb_store_bytes: -++ return &bpf_skb_store_bytes_proto; -++ case BPF_FUNC_csum_update: -++ return &bpf_csum_update_proto; -++ case BPF_FUNC_l3_csum_replace: -++ return &bpf_l3_csum_replace_proto; -++ case BPF_FUNC_l4_csum_replace: -++ return &bpf_l4_csum_replace_proto; -++ case BPF_FUNC_set_hash_invalid: -++ return &bpf_set_hash_invalid_proto; -++ case BPF_FUNC_lwt_push_encap: -++ return &bpf_lwt_xmit_push_encap_proto; -+ default: -+- return sk_filter_func_proto(func_id); -++ return lwt_out_func_proto(func_id, prog); -+ } -+ } -+ -+-static bool __is_valid_access(int off, int size, enum bpf_access_type type) -++static const struct bpf_func_proto * -++lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -+ { -+- /* check bounds */ -++ switch (func_id) { -++ default: -++ return lwt_out_func_proto(func_id, prog); -++ } -++} -++ -++static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ const int size_default = sizeof(__u32); -++ -+ if (off < 0 || off >= sizeof(struct __sk_buff)) -+ return false; -+ -+- /* disallow misaligned access */ -++ /* The verifier guarantees that size > 0. */ -+ if (off % size != 0) -+ return false; -+ -+- /* all __sk_buff fields are __u32 */ -+- if (size != 4) -++ switch (off) { -++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): -++ if (off + size > offsetofend(struct __sk_buff, cb[4])) -++ return false; -++ break; -++ case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]): -++ case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]): -++ case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4): -++ case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4): -++ case bpf_ctx_range(struct __sk_buff, data): -++ case bpf_ctx_range(struct __sk_buff, data_meta): -++ case bpf_ctx_range(struct __sk_buff, data_end): -++ if (size != size_default) -++ return false; -++ break; -++ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys): -+ return false; -++ case bpf_ctx_range(struct __sk_buff, tstamp): -++ if (size != sizeof(__u64)) -++ return false; -++ break; -++ case offsetof(struct __sk_buff, sk): -++ if (type == BPF_WRITE || size != sizeof(__u64)) -++ return false; -++ info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL; -++ break; -++ default: -++ /* Only narrow read access allowed for now. */ -++ if (type == BPF_WRITE) { -++ if (size != size_default) -++ return false; -++ } else { -++ bpf_ctx_record_field_size(info, size_default); -++ if (!bpf_ctx_narrow_access_ok(off, size, size_default)) -++ return false; -++ } -++ } -+ -+ return true; -+ } -+ -+ static bool sk_filter_is_valid_access(int off, int size, -+- enum bpf_access_type type) -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -+ { -+- if (off == offsetof(struct __sk_buff, tc_classid)) -++ return false; -++} -++ -++static bool lwt_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ switch (off) { -++ case bpf_ctx_range(struct __sk_buff, tc_classid): -++ case bpf_ctx_range_till(struct __sk_buff, family, local_port): -++ case bpf_ctx_range(struct __sk_buff, data_meta): -++ case bpf_ctx_range(struct __sk_buff, tstamp): -++ case bpf_ctx_range(struct __sk_buff, wire_len): -+ return false; -++ } -+ -+ if (type == BPF_WRITE) { -+ switch (off) { -+- case offsetof(struct __sk_buff, cb[0]) ... -+- offsetof(struct __sk_buff, cb[4]): -++ case bpf_ctx_range(struct __sk_buff, mark): -++ case bpf_ctx_range(struct __sk_buff, priority): -++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): -+ break; -+ default: -+ return false; -+ } -+ } -+ -+- return __is_valid_access(off, size, type); -++ switch (off) { -++ case bpf_ctx_range(struct __sk_buff, data): -++ info->reg_type = PTR_TO_PACKET; -++ break; -++ case bpf_ctx_range(struct __sk_buff, data_end): -++ info->reg_type = PTR_TO_PACKET_END; -++ break; -++ } -++ -++ return bpf_skb_is_valid_access(off, size, type, prog, info); -+ } -+ -+-static bool tc_cls_act_is_valid_access(int off, int size, -+- enum bpf_access_type type) -++ -++bool bpf_sock_common_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ switch (off) { -++ case bpf_ctx_range_till(struct bpf_sock, type, priority): -++ return false; -++ default: -++ return bpf_sock_is_valid_access(off, size, type, info); -++ } -++} -++ -++bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static bool sock_filter_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write, -++ const struct bpf_prog *prog) -++{ -++ /* Neither direct read nor direct write requires any preliminary -++ * action. -++ */ -++ return 0; -++} -++ -++static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write, -++ const struct bpf_prog *prog, int drop_verdict) -++{ -++ struct bpf_insn *insn = insn_buf; -++ -++ if (!direct_write) -++ return 0; -++ -++ /* if (!skb->cloned) -++ * goto start; -++ * -++ * (Fast-path, otherwise approximation that we might be -++ * a clone, do the rest in helper.) -++ */ -++ *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET()); -++ *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7); -++ -++ /* ret = bpf_skb_pull_data(skb, 0); */ -++ *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1); -++ *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2); -++ *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, -++ BPF_FUNC_skb_pull_data); -++ /* if (!ret) -++ * goto restore; -++ * return TC_ACT_SHOT; -++ */ -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2); -++ *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict); -++ *insn++ = BPF_EXIT_INSN(); -++ -++ /* restore: */ -++ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6); -++ /* start: */ -++ *insn++ = prog->insnsi[0]; -++ -++ return insn - insn_buf; -++} -++ -++static int bpf_gen_ld_abs(const struct bpf_insn *orig, -++ struct bpf_insn *insn_buf) -++{ -++ bool indirect = BPF_MODE(orig->code) == BPF_IND; -++ struct bpf_insn *insn = insn_buf; -++ -++ if (!indirect) { -++ *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm); -++ } else { -++ *insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg); -++ if (orig->imm) -++ *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm); -++ } -++ /* We're guaranteed here that CTX is in R6. */ -++ *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX); -++ -++ switch (BPF_SIZE(orig->code)) { -++ case BPF_B: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache); -++ break; -++ case BPF_H: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache); -++ break; -++ case BPF_W: -++ *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache); -++ break; -++ } -++ -++ *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2); -++ *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); -++ *insn++ = BPF_EXIT_INSN(); -++ -++ return insn - insn_buf; -++} -++ -++static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write, -++ const struct bpf_prog *prog) -+ { -+- if (off == offsetof(struct __sk_buff, tc_classid)) -+- return type == BPF_WRITE ? true : false; -++ return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT); -++} -+ -++static bool tc_cls_act_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -+ if (type == BPF_WRITE) { -+ switch (off) { -+- case offsetof(struct __sk_buff, mark): -+- case offsetof(struct __sk_buff, tc_index): -+- case offsetof(struct __sk_buff, priority): -+- case offsetof(struct __sk_buff, cb[0]) ... -+- offsetof(struct __sk_buff, cb[4]): -++ case bpf_ctx_range(struct __sk_buff, mark): -++ case bpf_ctx_range(struct __sk_buff, tc_index): -++ case bpf_ctx_range(struct __sk_buff, priority): -++ case bpf_ctx_range(struct __sk_buff, tc_classid): -++ case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]): -++ case bpf_ctx_range(struct __sk_buff, tstamp): -++ case bpf_ctx_range(struct __sk_buff, queue_mapping): -+ break; -+ default: -+ return false; -+ } -+ } -+- return __is_valid_access(off, size, type); -++ -++ switch (off) { -++ case bpf_ctx_range(struct __sk_buff, data): -++ info->reg_type = PTR_TO_PACKET; -++ break; -++ case bpf_ctx_range(struct __sk_buff, data_meta): -++ info->reg_type = PTR_TO_PACKET_META; -++ break; -++ case bpf_ctx_range(struct __sk_buff, data_end): -++ info->reg_type = PTR_TO_PACKET_END; -++ break; -++ case bpf_ctx_range_till(struct __sk_buff, family, local_port): -++ return false; -++ } -++ -++ return bpf_skb_is_valid_access(off, size, type, prog, info); -++} -++ -++static bool xdp_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++void bpf_warn_invalid_xdp_action(u32 act) -++{ -++} -++EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action); -++ -++static bool sock_addr_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static bool sock_ops_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write, -++ const struct bpf_prog *prog) -++{ -++ return bpf_unclone_prologue(insn_buf, direct_write, prog, SK_DROP); -++} -++ -++static bool sk_skb_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static bool sk_msg_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -++} -++ -++static bool flow_dissector_is_valid_access(int off, int size, -++ enum bpf_access_type type, -++ const struct bpf_prog *prog, -++ struct bpf_insn_access_aux *info) -++{ -++ return false; -+ } -+ -+-static u32 bpf_net_convert_ctx_access(enum bpf_access_type type, int dst_reg, -+- int src_reg, int ctx_off, -+- struct bpf_insn *insn_buf, -+- struct bpf_prog *prog) -++static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++ -++{ -++ return 0; -++} -++ -++static u32 bpf_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -+ { -+ struct bpf_insn *insn = insn_buf; -++ int off; -+ -+- switch (ctx_off) { -++ switch (si->off) { -+ case offsetof(struct __sk_buff, len): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); -+- -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, len)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, len, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, protocol): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); -+- -+- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, -+- offsetof(struct sk_buff, protocol)); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, protocol, 2, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, vlan_proto): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_proto) != 2); -+- -+- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, -+- offsetof(struct sk_buff, vlan_proto)); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, vlan_proto, 2, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, priority): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, priority) != 4); -+- -+ if (type == BPF_WRITE) -+- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, priority)); -++ *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, priority, 4, -++ target_size)); -+ else -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, priority)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, priority, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, ingress_ifindex): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, skb_iif) != 4); -+- -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, skb_iif)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, skb_iif, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, ifindex): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); -+- -+- *insn++ = BPF_LDX_MEM(bytes_to_bpf_size(FIELD_SIZEOF(struct sk_buff, dev)), -+- dst_reg, src_reg, -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), -++ si->dst_reg, si->src_reg, -+ offsetof(struct sk_buff, dev)); -+- *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1); -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, dst_reg, -+- offsetof(struct net_device, ifindex)); -++ *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct net_device, ifindex, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, hash): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); -+- -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, hash)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, hash, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, mark): -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); -+- -+ if (type == BPF_WRITE) -+- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, mark)); -++ *insn++ = BPF_STX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, mark, 4, -++ target_size)); -+ else -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, -+- offsetof(struct sk_buff, mark)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, mark, 4, -++ target_size)); -+ break; -+ -+ case offsetof(struct __sk_buff, pkt_type): -+- return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn); -++ *target_size = 1; -++ *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, -++ PKT_TYPE_OFFSET()); -++ *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX); -++#ifdef __BIG_ENDIAN_BITFIELD -++ *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5); -++#endif -++ break; -+ -+ case offsetof(struct __sk_buff, queue_mapping): -+- return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn); -++ if (type == BPF_WRITE) { -++ *insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, USHRT_MAX, 1); -++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, -++ queue_mapping, -++ 2, target_size)); -++ } else { -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, -++ queue_mapping, -++ 2, target_size)); -++ } -++ break; -+ -+ case offsetof(struct __sk_buff, vlan_present): -+- return convert_skb_access(SKF_AD_VLAN_TAG_PRESENT, -+- dst_reg, src_reg, insn); -++ *target_size = 1; -++ *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, -++ PKT_VLAN_PRESENT_OFFSET()); -++ if (PKT_VLAN_PRESENT_BIT) -++ *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT); -++ if (PKT_VLAN_PRESENT_BIT < 7) -++ *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, 1); -++ break; -+ -+ case offsetof(struct __sk_buff, vlan_tci): -+- return convert_skb_access(SKF_AD_VLAN_TAG, -+- dst_reg, src_reg, insn); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, vlan_tci, 2, -++ target_size)); -++ break; -+ -+ case offsetof(struct __sk_buff, cb[0]) ... -+- offsetof(struct __sk_buff, cb[4]): -++ offsetofend(struct __sk_buff, cb[4]) - 1: -+ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, data) < 20); -++ BUILD_BUG_ON((offsetof(struct sk_buff, cb) + -++ offsetof(struct qdisc_skb_cb, data)) % -++ sizeof(__u64)); -+ -+ prog->cb_access = 1; -+- ctx_off -= offsetof(struct __sk_buff, cb[0]); -+- ctx_off += offsetof(struct sk_buff, cb); -+- ctx_off += offsetof(struct qdisc_skb_cb, data); -++ off = si->off; -++ off -= offsetof(struct __sk_buff, cb[0]); -++ off += offsetof(struct sk_buff, cb); -++ off += offsetof(struct qdisc_skb_cb, data); -+ if (type == BPF_WRITE) -+- *insn++ = BPF_STX_MEM(BPF_W, dst_reg, src_reg, ctx_off); -++ *insn++ = BPF_STX_MEM(BPF_SIZE(si->code), si->dst_reg, -++ si->src_reg, off); -+ else -+- *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg, ctx_off); -++ *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg, -++ si->src_reg, off); -+ break; -+ -+ case offsetof(struct __sk_buff, tc_classid): -+- ctx_off -= offsetof(struct __sk_buff, tc_classid); -+- ctx_off += offsetof(struct sk_buff, cb); -+- ctx_off += offsetof(struct qdisc_skb_cb, tc_classid); -+- WARN_ON(type != BPF_WRITE); -+- *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, ctx_off); -++ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, tc_classid) != 2); -++ -++ off = si->off; -++ off -= offsetof(struct __sk_buff, tc_classid); -++ off += offsetof(struct sk_buff, cb); -++ off += offsetof(struct qdisc_skb_cb, tc_classid); -++ *target_size = 2; -++ if (type == BPF_WRITE) -++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, -++ si->src_reg, off); -++ else -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, -++ si->src_reg, off); -++ break; -++ -++ case offsetof(struct __sk_buff, data): -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, data)); -++ break; -++ -++ case offsetof(struct __sk_buff, data_meta): -++ off = si->off; -++ off -= offsetof(struct __sk_buff, data_meta); -++ off += offsetof(struct sk_buff, cb); -++ off += offsetof(struct bpf_skb_data_end, data_meta); -++ *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, -++ si->src_reg, off); -++ break; -++ -++ case offsetof(struct __sk_buff, data_end): -++ off = si->off; -++ off -= offsetof(struct __sk_buff, data_end); -++ off += offsetof(struct sk_buff, cb); -++ off += offsetof(struct bpf_skb_data_end, data_end); -++ *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, -++ si->src_reg, off); -+ break; -+ -+ case offsetof(struct __sk_buff, tc_index): -+ #ifdef CONFIG_NET_SCHED -+- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tc_index) != 2); -+- -+ if (type == BPF_WRITE) -+- *insn++ = BPF_STX_MEM(BPF_H, dst_reg, src_reg, -+- offsetof(struct sk_buff, tc_index)); -++ *insn++ = BPF_STX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, tc_index, 2, -++ target_size)); -+ else -+- *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg, -+- offsetof(struct sk_buff, tc_index)); -+- break; -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, tc_index, 2, -++ target_size)); -+ #else -++ *target_size = 2; -+ if (type == BPF_WRITE) -+- *insn++ = BPF_MOV64_REG(dst_reg, dst_reg); -++ *insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg); -+ else -+- *insn++ = BPF_MOV64_IMM(dst_reg, 0); -++ *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); -++#endif -++ break; -++ -++ case offsetof(struct __sk_buff, napi_id): -++ *target_size = 4; -++ *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); -++ break; -++ case offsetof(struct __sk_buff, family): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_family) != 2); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct sock_common, -++ skc_family, -++ 2, target_size)); -++ break; -++ case offsetof(struct __sk_buff, remote_ip4): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_daddr) != 4); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct sock_common, -++ skc_daddr, -++ 4, target_size)); -++ break; -++ case offsetof(struct __sk_buff, local_ip4): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, -++ skc_rcv_saddr) != 4); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct sock_common, -++ skc_rcv_saddr, -++ 4, target_size)); -++ break; -++ case offsetof(struct __sk_buff, remote_ip6[0]) ... -++ offsetof(struct __sk_buff, remote_ip6[3]): -++#if IS_ENABLED(CONFIG_IPV6) -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, -++ skc_v6_daddr.s6_addr32[0]) != 4); -++ -++ off = si->off; -++ off -= offsetof(struct __sk_buff, remote_ip6[0]); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ offsetof(struct sock_common, -++ skc_v6_daddr.s6_addr32[0]) + -++ off); -++#else -++ *insn++ = BPF_MOV32_IMM(si->dst_reg, 0); -++#endif -++ break; -++ case offsetof(struct __sk_buff, local_ip6[0]) ... -++ offsetof(struct __sk_buff, local_ip6[3]): -++#if IS_ENABLED(CONFIG_IPV6) -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, -++ skc_v6_rcv_saddr.s6_addr32[0]) != 4); -++ -++ off = si->off; -++ off -= offsetof(struct __sk_buff, local_ip6[0]); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ offsetof(struct sock_common, -++ skc_v6_rcv_saddr.s6_addr32[0]) + -++ off); -++#else -++ *insn++ = BPF_MOV32_IMM(si->dst_reg, 0); -++#endif -++ break; -++ -++ case offsetof(struct __sk_buff, remote_port): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_dport) != 2); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct sock_common, -++ skc_dport, -++ 2, target_size)); -++#ifndef __BIG_ENDIAN_BITFIELD -++ *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16); -++#endif -+ break; -++ -++ case offsetof(struct __sk_buff, local_port): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sock_common, skc_num) != 2); -++ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct sock_common, -++ skc_num, 2, target_size)); -++ break; -++ -++ case offsetof(struct __sk_buff, tstamp): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tstamp) != 8); -++ -++ if (type == BPF_WRITE) -++ *insn++ = BPF_STX_MEM(BPF_DW, -++ si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, -++ tstamp, 8, -++ target_size)); -++ else -++ *insn++ = BPF_LDX_MEM(BPF_DW, -++ si->dst_reg, si->src_reg, -++ bpf_target_off(struct sk_buff, -++ tstamp, 8, -++ target_size)); -++ break; -++ -++ case offsetof(struct __sk_buff, gso_segs): -++ /* si->dst_reg = skb_shinfo(SKB); */ -++#ifdef NET_SKBUFF_DATA_USES_OFFSET -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end), -++ BPF_REG_AX, si->src_reg, -++ offsetof(struct sk_buff, end)); -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, head)); -++ *insn++ = BPF_ALU64_REG(BPF_ADD, si->dst_reg, BPF_REG_AX); -++#else -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, end)); -+ #endif -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs), -++ si->dst_reg, si->dst_reg, -++ bpf_target_off(struct skb_shared_info, -++ gso_segs, 2, -++ target_size)); -++ break; -++ case offsetof(struct __sk_buff, wire_len): -++ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, pkt_len) != 4); -++ -++ off = si->off; -++ off -= offsetof(struct __sk_buff, wire_len); -++ off += offsetof(struct sk_buff, cb); -++ off += offsetof(struct qdisc_skb_cb, pkt_len); -++ *target_size = 4; -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off); -++ break; -++ -++ case offsetof(struct __sk_buff, sk): -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, sk)); -++ break; -+ } -+ -+ return insn - insn_buf; -+ } -+ -+-static const struct bpf_verifier_ops sk_filter_ops = { -+- .get_func_proto = sk_filter_func_proto, -+- .is_valid_access = sk_filter_is_valid_access, -+- .convert_ctx_access = bpf_net_convert_ctx_access, -++u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ struct bpf_insn *insn = insn_buf; -++ -++ switch (si->off) { -++ case offsetof(struct __sk_buff, ifindex): -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev), -++ si->dst_reg, si->src_reg, -++ offsetof(struct sk_buff, dev)); -++ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, -++ bpf_target_off(struct net_device, ifindex, 4, -++ target_size)); -++ break; -++ default: -++ return bpf_convert_ctx_access(type, si, insn_buf, prog, -++ target_size); -++ } -++ -++ return insn - insn_buf; -++} -++ -++static u32 xdp_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++/* SOCK_ADDR_LOAD_NESTED_FIELD() loads Nested Field S.F.NF where S is type of -++ * context Structure, F is Field in context structure that contains a pointer -++ * to Nested Structure of type NS that has the field NF. -++ * -++ * SIZE encodes the load size (BPF_B, BPF_H, etc). It's up to caller to make -++ * sure that SIZE is not greater than actual size of S.F.NF. -++ * -++ * If offset OFF is provided, the load happens from that offset relative to -++ * offset of NF. -++ */ -++#define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF) \ -++ do { \ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg, \ -++ si->src_reg, offsetof(S, F)); \ -++ *insn++ = BPF_LDX_MEM( \ -++ SIZE, si->dst_reg, si->dst_reg, \ -++ bpf_target_off(NS, NF, FIELD_SIZEOF(NS, NF), \ -++ target_size) \ -++ + OFF); \ -++ } while (0) -++ -++#define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF) \ -++ SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, \ -++ BPF_FIELD_SIZEOF(NS, NF), 0) -++ -++/* SOCK_ADDR_STORE_NESTED_FIELD_OFF() has semantic similar to -++ * SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF() but for store operation. -++ * -++ * In addition it uses Temporary Field TF (member of struct S) as the 3rd -++ * "register" since two registers available in convert_ctx_access are not -++ * enough: we can't override neither SRC, since it contains value to store, nor -++ * DST since it contains pointer to context that may be used by later -++ * instructions. But we need a temporary place to save pointer to nested -++ * structure whose field we want to store to. -++ */ -++#define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF) \ -++ do { \ -++ int tmp_reg = BPF_REG_9; \ -++ if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \ -++ --tmp_reg; \ -++ if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg) \ -++ --tmp_reg; \ -++ *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg, \ -++ offsetof(S, TF)); \ -++ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg, \ -++ si->dst_reg, offsetof(S, F)); \ -++ *insn++ = BPF_STX_MEM(SIZE, tmp_reg, si->src_reg, \ -++ bpf_target_off(NS, NF, FIELD_SIZEOF(NS, NF), \ -++ target_size) \ -++ + OFF); \ -++ *insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg, \ -++ offsetof(S, TF)); \ -++ } while (0) -++ -++#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \ -++ TF) \ -++ do { \ -++ if (type == BPF_WRITE) { \ -++ SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, \ -++ OFF, TF); \ -++ } else { \ -++ SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF( \ -++ S, NS, F, NF, SIZE, OFF); \ -++ } \ -++ } while (0) -++ -++#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD(S, NS, F, NF, TF) \ -++ SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF( \ -++ S, NS, F, NF, BPF_FIELD_SIZEOF(NS, NF), 0, TF) -++ -++static u32 sock_addr_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, -++ u32 *target_size) -++{ -++ return 0; -++} -++ -++static u32 sk_skb_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++static u32 sk_msg_convert_ctx_access(enum bpf_access_type type, -++ const struct bpf_insn *si, -++ struct bpf_insn *insn_buf, -++ struct bpf_prog *prog, u32 *target_size) -++{ -++ return 0; -++} -++ -++const struct bpf_verifier_ops sk_filter_verifier_ops = { -++ .is_valid_access = sk_filter_is_valid_access, -++ .convert_ctx_access = bpf_convert_ctx_access, -++ .gen_ld_abs = bpf_gen_ld_abs, -+ }; -+ -+-static const struct bpf_verifier_ops tc_cls_act_ops = { -+- .get_func_proto = tc_cls_act_func_proto, -+- .is_valid_access = tc_cls_act_is_valid_access, -+- .convert_ctx_access = bpf_net_convert_ctx_access, -++const struct bpf_prog_ops sk_filter_prog_ops = { -+ }; -+ -+-static struct bpf_prog_type_list sk_filter_type __read_mostly = { -+- .ops = &sk_filter_ops, -+- .type = BPF_PROG_TYPE_SOCKET_FILTER, -++const struct bpf_verifier_ops tc_cls_act_verifier_ops = { -++ .get_func_proto = tc_cls_act_func_proto, -++ .is_valid_access = tc_cls_act_is_valid_access, -++ .convert_ctx_access = tc_cls_act_convert_ctx_access, -++ .gen_prologue = tc_cls_act_prologue, -++ .gen_ld_abs = bpf_gen_ld_abs, -+ }; -+ -+-static struct bpf_prog_type_list sched_cls_type __read_mostly = { -+- .ops = &tc_cls_act_ops, -+- .type = BPF_PROG_TYPE_SCHED_CLS, -++const struct bpf_prog_ops tc_cls_act_prog_ops = { -+ }; -+ -+-static struct bpf_prog_type_list sched_act_type __read_mostly = { -+- .ops = &tc_cls_act_ops, -+- .type = BPF_PROG_TYPE_SCHED_ACT, -++const struct bpf_verifier_ops xdp_verifier_ops = { -++ .get_func_proto = xdp_func_proto, -++ .is_valid_access = xdp_is_valid_access, -++ .convert_ctx_access = xdp_convert_ctx_access, -++ .gen_prologue = bpf_noop_prologue, -+ }; -+ -+-static int __init register_sk_filter_ops(void) -+-{ -+- bpf_register_prog_type(&sk_filter_type); -+- bpf_register_prog_type(&sched_cls_type); -+- bpf_register_prog_type(&sched_act_type); -++const struct bpf_verifier_ops lwt_in_verifier_ops = { -++ .get_func_proto = lwt_in_func_proto, -++ .is_valid_access = lwt_is_valid_access, -++ .convert_ctx_access = bpf_convert_ctx_access, -++}; -+ -+- return 0; -+-} -+-late_initcall(register_sk_filter_ops); -++const struct bpf_prog_ops lwt_in_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops lwt_out_verifier_ops = { -++ .get_func_proto = lwt_out_func_proto, -++ .is_valid_access = lwt_is_valid_access, -++ .convert_ctx_access = bpf_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops lwt_out_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops lwt_xmit_verifier_ops = { -++ .get_func_proto = lwt_xmit_func_proto, -++ .is_valid_access = lwt_is_valid_access, -++ .convert_ctx_access = bpf_convert_ctx_access, -++ .gen_prologue = tc_cls_act_prologue, -++}; -++ -++const struct bpf_prog_ops lwt_xmit_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops lwt_seg6local_verifier_ops = { -++ .get_func_proto = lwt_seg6local_func_proto, -++ .is_valid_access = lwt_is_valid_access, -++ .convert_ctx_access = bpf_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops lwt_seg6local_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops cg_sock_verifier_ops = { -++ .get_func_proto = sock_filter_func_proto, -++ .is_valid_access = sock_filter_is_valid_access, -++ .convert_ctx_access = bpf_sock_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops cg_sock_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops cg_sock_addr_verifier_ops = { -++ .get_func_proto = sock_addr_func_proto, -++ .is_valid_access = sock_addr_is_valid_access, -++ .convert_ctx_access = sock_addr_convert_ctx_access, -++}; -+ -+-int __sk_detach_filter(struct sock *sk, bool locked) -++const struct bpf_prog_ops cg_sock_addr_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops sock_ops_verifier_ops = { -++ .get_func_proto = sock_ops_func_proto, -++ .is_valid_access = sock_ops_is_valid_access, -++ .convert_ctx_access = sock_ops_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops sock_ops_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops sk_skb_verifier_ops = { -++ .get_func_proto = sk_skb_func_proto, -++ .is_valid_access = sk_skb_is_valid_access, -++ .convert_ctx_access = sk_skb_convert_ctx_access, -++ .gen_prologue = sk_skb_prologue, -++}; -++ -++const struct bpf_prog_ops sk_skb_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops sk_msg_verifier_ops = { -++ .get_func_proto = sk_msg_func_proto, -++ .is_valid_access = sk_msg_is_valid_access, -++ .convert_ctx_access = sk_msg_convert_ctx_access, -++ .gen_prologue = bpf_noop_prologue, -++}; -++ -++const struct bpf_prog_ops sk_msg_prog_ops = { -++}; -++ -++const struct bpf_verifier_ops flow_dissector_verifier_ops = { -++ .get_func_proto = flow_dissector_func_proto, -++ .is_valid_access = flow_dissector_is_valid_access, -++ .convert_ctx_access = flow_dissector_convert_ctx_access, -++}; -++ -++const struct bpf_prog_ops flow_dissector_prog_ops = { -++}; -++ -++int sk_detach_filter(struct sock *sk) -+ { -+ int ret = -ENOENT; -+ struct sk_filter *filter; -+@@ -1928,7 +3664,8 @@ int __sk_detach_filter(struct sock *sk, -+ if (sock_flag(sk, SOCK_FILTER_LOCKED)) -+ return -EPERM; -+ -+- filter = rcu_dereference_protected(sk->sk_filter, locked); -++ filter = rcu_dereference_protected(sk->sk_filter, -++ lockdep_sock_is_held(sk)); -+ if (filter) { -+ RCU_INIT_POINTER(sk->sk_filter, NULL); -+ sk_filter_uncharge(sk, filter); -+@@ -1937,12 +3674,7 @@ int __sk_detach_filter(struct sock *sk, -+ -+ return ret; -+ } -+-EXPORT_SYMBOL_GPL(__sk_detach_filter); -+- -+-int sk_detach_filter(struct sock *sk) -+-{ -+- return __sk_detach_filter(sk, sock_owned_by_user(sk)); -+-} -++EXPORT_SYMBOL_GPL(sk_detach_filter); -+ -+ int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, -+ unsigned int len) -+@@ -1953,7 +3685,7 @@ int sk_get_filter(struct sock *sk, struc -+ -+ lock_sock(sk); -+ filter = rcu_dereference_protected(sk->sk_filter, -+- sock_owned_by_user(sk)); -++ lockdep_sock_is_held(sk)); -+ if (!filter) -+ goto out; -+ -+@@ -1987,3 +3719,5 @@ out: -+ release_sock(sk); -+ return ret; -+ } -++ -++ -+--- a/include/asm-generic/barrier.h -++++ b/include/asm-generic/barrier.h -+@@ -119,5 +119,29 @@ do { \ -+ ___p1; \ -+ }) -+ -++/** -++ * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees -++ * @ptr: pointer to the variable to wait on -++ * @cond: boolean expression to wait for -++ * -++ * Equivalent to using READ_ONCE() on the condition variable. -++ * -++ * Due to C lacking lambda expressions we load the value of *ptr into a -++ * pre-named variable @VAL to be used in @cond. -++ */ -++#ifndef smp_cond_load_relaxed -++#define smp_cond_load_relaxed(ptr, cond_expr) ({ \ -++ typeof(ptr) __PTR = (ptr); \ -++ typeof(*ptr) VAL; \ -++ for (;;) { \ -++ VAL = READ_ONCE(*__PTR); \ -++ if (cond_expr) \ -++ break; \ -++ cpu_relax(); \ -++ } \ -++ VAL; \ -++}) -++#endif -++ -+ #endif /* !__ASSEMBLY__ */ -+ #endif /* __ASM_GENERIC_BARRIER_H */ -+--- a/arch/arm/include/asm/barrier.h -++++ b/arch/arm/include/asm/barrier.h -+@@ -94,4 +94,6 @@ do { \ -+ #define smp_mb__after_atomic() smp_mb() -+ -+ #endif /* !__ASSEMBLY__ */ -++ -++#include -+ #endif /* __ASM_BARRIER_H */ -+--- a/include/linux/list_nulls.h -++++ b/include/linux/list_nulls.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -+ #ifndef _LINUX_LIST_NULLS_H -+ #define _LINUX_LIST_NULLS_H -+ -+@@ -29,6 +30,11 @@ struct hlist_nulls_node { -+ ((ptr)->first = (struct hlist_nulls_node *) NULLS_MARKER(nulls)) -+ -+ #define hlist_nulls_entry(ptr, type, member) container_of(ptr,type,member) -++ -++#define hlist_nulls_entry_safe(ptr, type, member) \ -++ ({ typeof(ptr) ____ptr = (ptr); \ -++ !is_a_nulls(____ptr) ? hlist_nulls_entry(____ptr, type, member) : NULL; \ -++ }) -+ /** -+ * ptr_is_a_nulls - Test if a ptr is a nulls -+ * @ptr: ptr to be tested -+@@ -57,7 +63,7 @@ static inline int hlist_nulls_unhashed(c -+ -+ static inline int hlist_nulls_empty(const struct hlist_nulls_head *h) -+ { -+- return is_a_nulls(h->first); -++ return is_a_nulls(READ_ONCE(h->first)); -+ } -+ -+ static inline void hlist_nulls_add_head(struct hlist_nulls_node *n, -+@@ -66,10 +72,10 @@ static inline void hlist_nulls_add_head( -+ struct hlist_nulls_node *first = h->first; -+ -+ n->next = first; -+- n->pprev = &h->first; -++ WRITE_ONCE(n->pprev, &h->first); -+ h->first = n; -+ if (!is_a_nulls(first)) -+- first->pprev = &n->next; -++ WRITE_ONCE(first->pprev, &n->next); -+ } -+ -+ static inline void __hlist_nulls_del(struct hlist_nulls_node *n) -+@@ -79,13 +85,13 @@ static inline void __hlist_nulls_del(str -+ -+ WRITE_ONCE(*pprev, next); -+ if (!is_a_nulls(next)) -+- next->pprev = pprev; -++ WRITE_ONCE(next->pprev, pprev); -+ } -+ -+ static inline void hlist_nulls_del(struct hlist_nulls_node *n) -+ { -+ __hlist_nulls_del(n); -+- n->pprev = LIST_POISON2; -++ WRITE_ONCE(n->pprev, LIST_POISON2); -+ } -+ -+ /** -+--- a/include/linux/rculist_nulls.h -++++ b/include/linux/rculist_nulls.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -+ #ifndef _LINUX_RCULIST_NULLS_H -+ #define _LINUX_RCULIST_NULLS_H -+ -+@@ -33,7 +34,7 @@ static inline void hlist_nulls_del_init_ -+ { -+ if (!hlist_nulls_unhashed(n)) { -+ __hlist_nulls_del(n); -+- n->pprev = NULL; -++ WRITE_ONCE(n->pprev, NULL); -+ } -+ } -+ -+@@ -65,7 +66,7 @@ static inline void hlist_nulls_del_init_ -+ static inline void hlist_nulls_del_rcu(struct hlist_nulls_node *n) -+ { -+ __hlist_nulls_del(n); -+- n->pprev = LIST_POISON2; -++ WRITE_ONCE(n->pprev, LIST_POISON2); -+ } -+ -+ /** -+@@ -93,11 +94,49 @@ static inline void hlist_nulls_add_head_ -+ struct hlist_nulls_node *first = h->first; -+ -+ n->next = first; -+- n->pprev = &h->first; -++ WRITE_ONCE(n->pprev, &h->first); -+ rcu_assign_pointer(hlist_nulls_first_rcu(h), n); -+ if (!is_a_nulls(first)) -+- first->pprev = &n->next; -++ WRITE_ONCE(first->pprev, &n->next); -+ } -++ -++/** -++ * hlist_nulls_add_tail_rcu -++ * @n: the element to add to the hash list. -++ * @h: the list to add to. -++ * -++ * Description: -++ * Adds the specified element to the specified hlist_nulls, -++ * while permitting racing traversals. -++ * -++ * The caller must take whatever precautions are necessary -++ * (such as holding appropriate locks) to avoid racing -++ * with another list-mutation primitive, such as hlist_nulls_add_head_rcu() -++ * or hlist_nulls_del_rcu(), running on this same list. -++ * However, it is perfectly legal to run concurrently with -++ * the _rcu list-traversal primitives, such as -++ * hlist_nulls_for_each_entry_rcu(), used to prevent memory-consistency -++ * problems on Alpha CPUs. Regardless of the type of CPU, the -++ * list-traversal primitive must be guarded by rcu_read_lock(). -++ */ -++static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, -++ struct hlist_nulls_head *h) -++{ -++ struct hlist_nulls_node *i, *last = NULL; -++ -++ /* Note: write side code, so rcu accessors are not needed. */ -++ for (i = h->first; !is_a_nulls(i); i = i->next) -++ last = i; -++ -++ if (last) { -++ n->next = last->next; -++ n->pprev = &last->next; -++ rcu_assign_pointer(hlist_next_rcu(last), n); -++ } else { -++ hlist_nulls_add_head_rcu(n, h); -++ } -++} -++ -+ /** -+ * hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type -+ * @tpos: the type * to use as a loop cursor. -+@@ -107,7 +146,7 @@ static inline void hlist_nulls_add_head_ -+ * -+ * The barrier() is needed to make sure compiler doesn't cache first element [1], -+ * as this loop can be restarted [2] -+- * [1] Documentation/atomic_ops.txt around line 114 -++ * [1] Documentation/core-api/atomic_ops.rst around line 114 -+ * [2] Documentation/RCU/rculist_nulls.txt around line 146 -+ */ -+ #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) \ -+@@ -117,5 +156,19 @@ static inline void hlist_nulls_add_head_ -+ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; }); \ -+ pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos))) -+ -++/** -++ * hlist_nulls_for_each_entry_safe - -++ * iterate over list of given type safe against removal of list entry -++ * @tpos: the type * to use as a loop cursor. -++ * @pos: the &struct hlist_nulls_node to use as a loop cursor. -++ * @head: the head for your list. -++ * @member: the name of the hlist_nulls_node within the struct. -++ */ -++#define hlist_nulls_for_each_entry_safe(tpos, pos, head, member) \ -++ for (({barrier();}), \ -++ pos = rcu_dereference_raw(hlist_nulls_first_rcu(head)); \ -++ (!is_a_nulls(pos)) && \ -++ ({ tpos = hlist_nulls_entry(pos, typeof(*tpos), member); \ -++ pos = rcu_dereference_raw(hlist_nulls_next_rcu(pos)); 1; });) -+ #endif -+ #endif -+--- a/drivers/net/tun.c -++++ b/drivers/net/tun.c -+@@ -627,8 +627,9 @@ static int tun_attach(struct tun_struct -+ -+ /* Re-attach the filter to persist device */ -+ if (!skip_filter && (tun->filter_attached == true)) { -+- err = __sk_attach_filter(&tun->fprog, tfile->socket.sk, -+- lockdep_rtnl_is_held()); -++ lock_sock(tfile->socket.sk); -++ err = sk_attach_filter(&tun->fprog, tfile->socket.sk); -++ release_sock(tfile->socket.sk); -+ if (!err) -+ goto out; -+ } -+@@ -1835,7 +1836,9 @@ static void tun_detach_filter(struct tun -+ -+ for (i = 0; i < n; i++) { -+ tfile = rtnl_dereference(tun->tfiles[i]); -+- __sk_detach_filter(tfile->socket.sk, lockdep_rtnl_is_held()); -++ lock_sock(tfile->socket.sk); -++ sk_detach_filter(tfile->socket.sk); -++ release_sock(tfile->socket.sk); -+ } -+ -+ tun->filter_attached = false; -+@@ -1848,8 +1851,9 @@ static int tun_attach_filter(struct tun_ -+ -+ for (i = 0; i < tun->numqueues; i++) { -+ tfile = rtnl_dereference(tun->tfiles[i]); -+- ret = __sk_attach_filter(&tun->fprog, tfile->socket.sk, -+- lockdep_rtnl_is_held()); -++ lock_sock(tfile->socket.sk); -++ ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); -++ release_sock(tfile->socket.sk); -+ if (ret) { -+ tun_detach_filter(tun, i); -+ return ret; -+--- a/include/linux/list.h -++++ b/include/linux/list.h -+@@ -1,3 +1,4 @@ -++/* SPDX-License-Identifier: GPL-2.0 */ -+ #ifndef _LINUX_LIST_H -+ #define _LINUX_LIST_H -+ -+@@ -24,31 +25,46 @@ -+ -+ static inline void INIT_LIST_HEAD(struct list_head *list) -+ { -+- list->next = list; -++ WRITE_ONCE(list->next, list); -+ list->prev = list; -+ } -+ -++#ifdef CONFIG_DEBUG_LIST -++extern bool __list_add_valid(struct list_head *new, -++ struct list_head *prev, -++ struct list_head *next); -++extern bool __list_del_entry_valid(struct list_head *entry); -++#else -++static inline bool __list_add_valid(struct list_head *new, -++ struct list_head *prev, -++ struct list_head *next) -++{ -++ return true; -++} -++static inline bool __list_del_entry_valid(struct list_head *entry) -++{ -++ return true; -++} -++#endif -++ -+ /* -+ * Insert a new entry between two known consecutive entries. -+ * -+ * This is only for internal list manipulation where we know -+ * the prev/next entries already! -+ */ -+-#ifndef CONFIG_DEBUG_LIST -+ static inline void __list_add(struct list_head *new, -+ struct list_head *prev, -+ struct list_head *next) -+ { -++ if (!__list_add_valid(new, prev, next)) -++ return; -++ -+ next->prev = new; -+ new->next = next; -+ new->prev = prev; -+- prev->next = new; -++ WRITE_ONCE(prev->next, new); -+ } -+-#else -+-extern void __list_add(struct list_head *new, -+- struct list_head *prev, -+- struct list_head *next); -+-#endif -+ -+ /** -+ * list_add - add a new entry -+@@ -90,28 +106,40 @@ static inline void __list_del(struct lis -+ WRITE_ONCE(prev->next, next); -+ } -+ -++/* -++ * Delete a list entry and clear the 'prev' pointer. -++ * -++ * This is a special-purpose list clearing method used in the networking code -++ * for lists allocated as per-cpu, where we don't want to incur the extra -++ * WRITE_ONCE() overhead of a regular list_del_init(). The code that uses this -++ * needs to check the node 'prev' pointer instead of calling list_empty(). -++ */ -++static inline void __list_del_clearprev(struct list_head *entry) -++{ -++ __list_del(entry->prev, entry->next); -++ entry->prev = NULL; -++} -++ -+ /** -+ * list_del - deletes entry from list. -+ * @entry: the element to delete from the list. -+ * Note: list_empty() on entry does not return true after this, the entry is -+ * in an undefined state. -+ */ -+-#ifndef CONFIG_DEBUG_LIST -+ static inline void __list_del_entry(struct list_head *entry) -+ { -++ if (!__list_del_entry_valid(entry)) -++ return; -++ -+ __list_del(entry->prev, entry->next); -+ } -+ -+ static inline void list_del(struct list_head *entry) -+ { -+- __list_del(entry->prev, entry->next); -++ __list_del_entry(entry); -+ entry->next = LIST_POISON1; -+ entry->prev = LIST_POISON2; -+ } -+-#else -+-extern void __list_del_entry(struct list_head *entry); -+-extern void list_del(struct list_head *entry); -+-#endif -+ -+ /** -+ * list_replace - replace old entry by new one -+@@ -137,6 +165,23 @@ static inline void list_replace_init(str -+ } -+ -+ /** -++ * list_swap - replace entry1 with entry2 and re-add entry1 at entry2's position -++ * @entry1: the location to place entry2 -++ * @entry2: the location to place entry1 -++ */ -++static inline void list_swap(struct list_head *entry1, -++ struct list_head *entry2) -++{ -++ struct list_head *pos = entry2->prev; -++ -++ list_del(entry2); -++ list_replace(entry1, entry2); -++ if (pos == entry1) -++ pos = entry2; -++ list_add(entry1, pos); -++} -++ -++/** -+ * list_del_init - deletes entry from list and reinitialize it. -+ * @entry: the element to delete from the list. -+ */ -+@@ -170,6 +215,40 @@ static inline void list_move_tail(struct -+ } -+ -+ /** -++ * list_bulk_move_tail - move a subsection of a list to its tail -++ * @head: the head that will follow our entry -++ * @first: first entry to move -++ * @last: last entry to move, can be the same as first -++ * -++ * Move all entries between @first and including @last before @head. -++ * All three entries must belong to the same linked list. -++ */ -++static inline void list_bulk_move_tail(struct list_head *head, -++ struct list_head *first, -++ struct list_head *last) -++{ -++ first->prev->next = last->next; -++ last->next->prev = first->prev; -++ -++ head->prev->next = first; -++ first->prev = head->prev; -++ -++ last->next = head; -++ head->prev = last; -++} -++ -++/** -++ * list_is_first -- tests whether @list is the first entry in list @head -++ * @list: the entry to test -++ * @head: the head of the list -++ */ -++static inline int list_is_first(const struct list_head *list, -++ const struct list_head *head) -++{ -++ return list->prev == head; -++} -++ -++/** -+ * list_is_last - tests whether @list is the last entry in list @head -+ * @list: the entry to test -+ * @head: the head of the list -+@@ -186,7 +265,7 @@ static inline int list_is_last(const str -+ */ -+ static inline int list_empty(const struct list_head *head) -+ { -+- return head->next == head; -++ return READ_ONCE(head->next) == head; -+ } -+ -+ /** -+@@ -223,6 +302,24 @@ static inline void list_rotate_left(stru -+ } -+ -+ /** -++ * list_rotate_to_front() - Rotate list to specific item. -++ * @list: The desired new front of the list. -++ * @head: The head of the list. -++ * -++ * Rotates list so that @list becomes the new front of the list. -++ */ -++static inline void list_rotate_to_front(struct list_head *list, -++ struct list_head *head) -++{ -++ /* -++ * Deletes the list head from the list denoted by @head and -++ * places it as the tail of @list, this effectively rotates the -++ * list so that @list is at the front. -++ */ -++ list_move_tail(head, list); -++} -++ -++/** -+ * list_is_singular - tests whether a list has just one entry. -+ * @head: the list to test. -+ */ -+@@ -271,6 +368,36 @@ static inline void list_cut_position(str -+ __list_cut_position(list, head, entry); -+ } -+ -++/** -++ * list_cut_before - cut a list into two, before given entry -++ * @list: a new list to add all removed entries -++ * @head: a list with entries -++ * @entry: an entry within head, could be the head itself -++ * -++ * This helper moves the initial part of @head, up to but -++ * excluding @entry, from @head to @list. You should pass -++ * in @entry an element you know is on @head. @list should -++ * be an empty list or a list you do not care about losing -++ * its data. -++ * If @entry == @head, all entries on @head are moved to -++ * @list. -++ */ -++static inline void list_cut_before(struct list_head *list, -++ struct list_head *head, -++ struct list_head *entry) -++{ -++ if (head->next == entry) { -++ INIT_LIST_HEAD(list); -++ return; -++ } -++ list->next = head->next; -++ list->next->prev = list; -++ list->prev = entry->prev; -++ list->prev->next = list; -++ head->next = entry; -++ entry->prev = head; -++} -++ -+ static inline void __list_splice(const struct list_head *list, -+ struct list_head *prev, -+ struct list_head *next) -+@@ -381,8 +508,11 @@ static inline void list_splice_tail_init -+ * -+ * Note that if the list is empty, it returns NULL. -+ */ -+-#define list_first_entry_or_null(ptr, type, member) \ -+- (!list_empty(ptr) ? list_first_entry(ptr, type, member) : NULL) -++#define list_first_entry_or_null(ptr, type, member) ({ \ -++ struct list_head *head__ = (ptr); \ -++ struct list_head *pos__ = READ_ONCE(head__->next); \ -++ pos__ != head__ ? list_entry(pos__, type, member) : NULL; \ -++}) -+ -+ /** -+ * list_next_entry - get the next element in list -+@@ -511,6 +641,19 @@ static inline void list_splice_tail_init -+ pos = list_next_entry(pos, member)) -+ -+ /** -++ * list_for_each_entry_from_reverse - iterate backwards over list of given type -++ * from the current point -++ * @pos: the type * to use as a loop cursor. -++ * @head: the head for your list. -++ * @member: the name of the list_head within the struct. -++ * -++ * Iterate backwards over list of given type, continuing from current position. -++ */ -++#define list_for_each_entry_from_reverse(pos, head, member) \ -++ for (; &pos->member != (head); \ -++ pos = list_prev_entry(pos, member)) -++ -++/** -+ * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry -+ * @pos: the type * to use as a loop cursor. -+ * @n: another type * to use as temporary storage -+@@ -608,7 +751,7 @@ static inline int hlist_unhashed(const s -+ -+ static inline int hlist_empty(const struct hlist_head *h) -+ { -+- return !h->first; -++ return !READ_ONCE(h->first); -+ } -+ -+ static inline void __hlist_del(struct hlist_node *n) -+@@ -642,7 +785,7 @@ static inline void hlist_add_head(struct -+ n->next = first; -+ if (first) -+ first->pprev = &n->next; -+- h->first = n; -++ WRITE_ONCE(h->first, n); -+ n->pprev = &h->first; -+ } -+ -+@@ -653,7 +796,7 @@ static inline void hlist_add_before(stru -+ n->pprev = next->pprev; -+ n->next = next; -+ next->pprev = &n->next; -+- *(n->pprev) = n; -++ WRITE_ONCE(*(n->pprev), n); -+ } -+ -+ static inline void hlist_add_behind(struct hlist_node *n, -+@@ -679,6 +822,16 @@ static inline bool hlist_fake(struct hli -+ } -+ -+ /* -++ * Check whether the node is the only node of the head without -++ * accessing head: -++ */ -++static inline bool -++hlist_is_singular_node(struct hlist_node *n, struct hlist_head *h) -++{ -++ return !n->next && n->pprev == &h->first; -++} -++ -++/* -+ * Move a list from one list head to another. Fixup the pprev -+ * reference of the first entry if it exists. -+ */ -+--- /dev/null -++++ b/include/linux/ptr_ring.h -+@@ -0,0 +1,673 @@ -++/* SPDX-License-Identifier: GPL-2.0-or-later */ -++/* -++ * Definitions for the 'struct ptr_ring' datastructure. -++ * -++ * Author: -++ * Michael S. Tsirkin -++ * -++ * Copyright (C) 2016 Red Hat, Inc. -++ * -++ * This is a limited-size FIFO maintaining pointers in FIFO order, with -++ * one CPU producing entries and another consuming entries from a FIFO. -++ * -++ * This implementation tries to minimize cache-contention when there is a -++ * single producer and a single consumer CPU. -++ */ -++ -++#ifndef _LINUX_PTR_RING_H -++#define _LINUX_PTR_RING_H 1 -++ -++#ifdef __KERNEL__ -++#include -++#include -++#include -++#include -++#include -++#include -++#endif -++ -++struct ptr_ring { -++ int producer ____cacheline_aligned_in_smp; -++ spinlock_t producer_lock; -++ int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */ -++ int consumer_tail; /* next entry to invalidate */ -++ spinlock_t consumer_lock; -++ /* Shared consumer/producer data */ -++ /* Read-only by both the producer and the consumer */ -++ int size ____cacheline_aligned_in_smp; /* max entries in queue */ -++ int batch; /* number of entries to consume in a batch */ -++ void **queue; -++}; -++ -++/* Note: callers invoking this in a loop must use a compiler barrier, -++ * for example cpu_relax(). -++ * -++ * NB: this is unlike __ptr_ring_empty in that callers must hold producer_lock: -++ * see e.g. ptr_ring_full. -++ */ -++static inline bool __ptr_ring_full(struct ptr_ring *r) -++{ -++ return r->queue[r->producer]; -++} -++ -++static inline bool ptr_ring_full(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock(&r->producer_lock); -++ ret = __ptr_ring_full(r); -++ spin_unlock(&r->producer_lock); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_full_irq(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock_irq(&r->producer_lock); -++ ret = __ptr_ring_full(r); -++ spin_unlock_irq(&r->producer_lock); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_full_any(struct ptr_ring *r) -++{ -++ unsigned long flags; -++ bool ret; -++ -++ spin_lock_irqsave(&r->producer_lock, flags); -++ ret = __ptr_ring_full(r); -++ spin_unlock_irqrestore(&r->producer_lock, flags); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_full_bh(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock_bh(&r->producer_lock); -++ ret = __ptr_ring_full(r); -++ spin_unlock_bh(&r->producer_lock); -++ -++ return ret; -++} -++ -++/* Note: callers invoking this in a loop must use a compiler barrier, -++ * for example cpu_relax(). Callers must hold producer_lock. -++ * Callers are responsible for making sure pointer that is being queued -++ * points to a valid data. -++ */ -++static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) -++{ -++ if (unlikely(!r->size) || r->queue[r->producer]) -++ return -ENOSPC; -++ -++ /* Make sure the pointer we are storing points to a valid data. */ -++ /* Pairs with smp_read_barrier_depends in __ptr_ring_consume. */ -++ smp_wmb(); -++ -++ WRITE_ONCE(r->queue[r->producer++], ptr); -++ if (unlikely(r->producer >= r->size)) -++ r->producer = 0; -++ return 0; -++} -++ -++/* -++ * Note: resize (below) nests producer lock within consumer lock, so if you -++ * consume in interrupt or BH context, you must disable interrupts/BH when -++ * calling this. -++ */ -++static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr) -++{ -++ int ret; -++ -++ spin_lock(&r->producer_lock); -++ ret = __ptr_ring_produce(r, ptr); -++ spin_unlock(&r->producer_lock); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr) -++{ -++ int ret; -++ -++ spin_lock_irq(&r->producer_lock); -++ ret = __ptr_ring_produce(r, ptr); -++ spin_unlock_irq(&r->producer_lock); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr) -++{ -++ unsigned long flags; -++ int ret; -++ -++ spin_lock_irqsave(&r->producer_lock, flags); -++ ret = __ptr_ring_produce(r, ptr); -++ spin_unlock_irqrestore(&r->producer_lock, flags); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr) -++{ -++ int ret; -++ -++ spin_lock_bh(&r->producer_lock); -++ ret = __ptr_ring_produce(r, ptr); -++ spin_unlock_bh(&r->producer_lock); -++ -++ return ret; -++} -++ -++static inline void *__ptr_ring_peek(struct ptr_ring *r) -++{ -++ if (likely(r->size)) -++ return READ_ONCE(r->queue[r->consumer_head]); -++ return NULL; -++} -++ -++/* -++ * Test ring empty status without taking any locks. -++ * -++ * NB: This is only safe to call if ring is never resized. -++ * -++ * However, if some other CPU consumes ring entries at the same time, the value -++ * returned is not guaranteed to be correct. -++ * -++ * In this case - to avoid incorrectly detecting the ring -++ * as empty - the CPU consuming the ring entries is responsible -++ * for either consuming all ring entries until the ring is empty, -++ * or synchronizing with some other CPU and causing it to -++ * re-test __ptr_ring_empty and/or consume the ring enteries -++ * after the synchronization point. -++ * -++ * Note: callers invoking this in a loop must use a compiler barrier, -++ * for example cpu_relax(). -++ */ -++static inline bool __ptr_ring_empty(struct ptr_ring *r) -++{ -++ if (likely(r->size)) -++ return !r->queue[READ_ONCE(r->consumer_head)]; -++ return true; -++} -++ -++static inline bool ptr_ring_empty(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock(&r->consumer_lock); -++ ret = __ptr_ring_empty(r); -++ spin_unlock(&r->consumer_lock); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_empty_irq(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock_irq(&r->consumer_lock); -++ ret = __ptr_ring_empty(r); -++ spin_unlock_irq(&r->consumer_lock); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_empty_any(struct ptr_ring *r) -++{ -++ unsigned long flags; -++ bool ret; -++ -++ spin_lock_irqsave(&r->consumer_lock, flags); -++ ret = __ptr_ring_empty(r); -++ spin_unlock_irqrestore(&r->consumer_lock, flags); -++ -++ return ret; -++} -++ -++static inline bool ptr_ring_empty_bh(struct ptr_ring *r) -++{ -++ bool ret; -++ -++ spin_lock_bh(&r->consumer_lock); -++ ret = __ptr_ring_empty(r); -++ spin_unlock_bh(&r->consumer_lock); -++ -++ return ret; -++} -++ -++/* Must only be called after __ptr_ring_peek returned !NULL */ -++static inline void __ptr_ring_discard_one(struct ptr_ring *r) -++{ -++ /* Fundamentally, what we want to do is update consumer -++ * index and zero out the entry so producer can reuse it. -++ * Doing it naively at each consume would be as simple as: -++ * consumer = r->consumer; -++ * r->queue[consumer++] = NULL; -++ * if (unlikely(consumer >= r->size)) -++ * consumer = 0; -++ * r->consumer = consumer; -++ * but that is suboptimal when the ring is full as producer is writing -++ * out new entries in the same cache line. Defer these updates until a -++ * batch of entries has been consumed. -++ */ -++ /* Note: we must keep consumer_head valid at all times for __ptr_ring_empty -++ * to work correctly. -++ */ -++ int consumer_head = r->consumer_head; -++ int head = consumer_head++; -++ -++ /* Once we have processed enough entries invalidate them in -++ * the ring all at once so producer can reuse their space in the ring. -++ * We also do this when we reach end of the ring - not mandatory -++ * but helps keep the implementation simple. -++ */ -++ if (unlikely(consumer_head - r->consumer_tail >= r->batch || -++ consumer_head >= r->size)) { -++ /* Zero out entries in the reverse order: this way we touch the -++ * cache line that producer might currently be reading the last; -++ * producer won't make progress and touch other cache lines -++ * besides the first one until we write out all entries. -++ */ -++ while (likely(head >= r->consumer_tail)) -++ r->queue[head--] = NULL; -++ r->consumer_tail = consumer_head; -++ } -++ if (unlikely(consumer_head >= r->size)) { -++ consumer_head = 0; -++ r->consumer_tail = 0; -++ } -++ /* matching READ_ONCE in __ptr_ring_empty for lockless tests */ -++ WRITE_ONCE(r->consumer_head, consumer_head); -++} -++ -++static inline void *__ptr_ring_consume(struct ptr_ring *r) -++{ -++ void *ptr; -++ -++ /* The READ_ONCE in __ptr_ring_peek guarantees that anyone -++ * accessing data through the pointer is up to date. Pairs -++ * with smp_wmb in __ptr_ring_produce. -++ */ -++ ptr = __ptr_ring_peek(r); -++ if (ptr) -++ __ptr_ring_discard_one(r); -++ -++ return ptr; -++} -++ -++static inline int __ptr_ring_consume_batched(struct ptr_ring *r, -++ void **array, int n) -++{ -++ void *ptr; -++ int i; -++ -++ for (i = 0; i < n; i++) { -++ ptr = __ptr_ring_consume(r); -++ if (!ptr) -++ break; -++ array[i] = ptr; -++ } -++ -++ return i; -++} -++ -++/* -++ * Note: resize (below) nests producer lock within consumer lock, so if you -++ * call this in interrupt or BH context, you must disable interrupts/BH when -++ * producing. -++ */ -++static inline void *ptr_ring_consume(struct ptr_ring *r) -++{ -++ void *ptr; -++ -++ spin_lock(&r->consumer_lock); -++ ptr = __ptr_ring_consume(r); -++ spin_unlock(&r->consumer_lock); -++ -++ return ptr; -++} -++ -++static inline void *ptr_ring_consume_irq(struct ptr_ring *r) -++{ -++ void *ptr; -++ -++ spin_lock_irq(&r->consumer_lock); -++ ptr = __ptr_ring_consume(r); -++ spin_unlock_irq(&r->consumer_lock); -++ -++ return ptr; -++} -++ -++static inline void *ptr_ring_consume_any(struct ptr_ring *r) -++{ -++ unsigned long flags; -++ void *ptr; -++ -++ spin_lock_irqsave(&r->consumer_lock, flags); -++ ptr = __ptr_ring_consume(r); -++ spin_unlock_irqrestore(&r->consumer_lock, flags); -++ -++ return ptr; -++} -++ -++static inline void *ptr_ring_consume_bh(struct ptr_ring *r) -++{ -++ void *ptr; -++ -++ spin_lock_bh(&r->consumer_lock); -++ ptr = __ptr_ring_consume(r); -++ spin_unlock_bh(&r->consumer_lock); -++ -++ return ptr; -++} -++ -++static inline int ptr_ring_consume_batched(struct ptr_ring *r, -++ void **array, int n) -++{ -++ int ret; -++ -++ spin_lock(&r->consumer_lock); -++ ret = __ptr_ring_consume_batched(r, array, n); -++ spin_unlock(&r->consumer_lock); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r, -++ void **array, int n) -++{ -++ int ret; -++ -++ spin_lock_irq(&r->consumer_lock); -++ ret = __ptr_ring_consume_batched(r, array, n); -++ spin_unlock_irq(&r->consumer_lock); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_consume_batched_any(struct ptr_ring *r, -++ void **array, int n) -++{ -++ unsigned long flags; -++ int ret; -++ -++ spin_lock_irqsave(&r->consumer_lock, flags); -++ ret = __ptr_ring_consume_batched(r, array, n); -++ spin_unlock_irqrestore(&r->consumer_lock, flags); -++ -++ return ret; -++} -++ -++static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, -++ void **array, int n) -++{ -++ int ret; -++ -++ spin_lock_bh(&r->consumer_lock); -++ ret = __ptr_ring_consume_batched(r, array, n); -++ spin_unlock_bh(&r->consumer_lock); -++ -++ return ret; -++} -++ -++/* Cast to structure type and call a function without discarding from FIFO. -++ * Function must return a value. -++ * Callers must take consumer_lock. -++ */ -++#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r))) -++ -++#define PTR_RING_PEEK_CALL(r, f) ({ \ -++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ -++ \ -++ spin_lock(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ -++ spin_unlock(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v; \ -++}) -++ -++#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \ -++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ -++ \ -++ spin_lock_irq(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ -++ spin_unlock_irq(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v; \ -++}) -++ -++#define PTR_RING_PEEK_CALL_BH(r, f) ({ \ -++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ -++ \ -++ spin_lock_bh(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ -++ spin_unlock_bh(&(r)->consumer_lock); \ -++ __PTR_RING_PEEK_CALL_v; \ -++}) -++ -++#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \ -++ typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ -++ unsigned long __PTR_RING_PEEK_CALL_f;\ -++ \ -++ spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ -++ __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ -++ spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ -++ __PTR_RING_PEEK_CALL_v; \ -++}) -++ -++/* Not all gfp_t flags (besides GFP_KERNEL) are allowed. See -++ * documentation for vmalloc for which of them are legal. -++ */ -++static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) -++{ -++ if (size > KMALLOC_MAX_SIZE / sizeof(void *)) -++ return NULL; -++ return kmalloc_array(size, sizeof(void *), gfp | __GFP_ZERO); -++} -++ -++static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) -++{ -++ r->size = size; -++ r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue)); -++ /* We need to set batch at least to 1 to make logic -++ * in __ptr_ring_discard_one work correctly. -++ * Batching too much (because ring is small) would cause a lot of -++ * burstiness. Needs tuning, for now disable batching. -++ */ -++ if (r->batch > r->size / 2 || !r->batch) -++ r->batch = 1; -++} -++ -++static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) -++{ -++ r->queue = __ptr_ring_init_queue_alloc(size, gfp); -++ if (!r->queue) -++ return -ENOMEM; -++ -++ __ptr_ring_set_size(r, size); -++ r->producer = r->consumer_head = r->consumer_tail = 0; -++ spin_lock_init(&r->producer_lock); -++ spin_lock_init(&r->consumer_lock); -++ -++ return 0; -++} -++ -++/* -++ * Return entries into ring. Destroy entries that don't fit. -++ * -++ * Note: this is expected to be a rare slow path operation. -++ * -++ * Note: producer lock is nested within consumer lock, so if you -++ * resize you must make sure all uses nest correctly. -++ * In particular if you consume ring in interrupt or BH context, you must -++ * disable interrupts/BH when doing so. -++ */ -++static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n, -++ void (*destroy)(void *)) -++{ -++ unsigned long flags; -++ int head; -++ -++ spin_lock_irqsave(&r->consumer_lock, flags); -++ spin_lock(&r->producer_lock); -++ -++ if (!r->size) -++ goto done; -++ -++ /* -++ * Clean out buffered entries (for simplicity). This way following code -++ * can test entries for NULL and if not assume they are valid. -++ */ -++ head = r->consumer_head - 1; -++ while (likely(head >= r->consumer_tail)) -++ r->queue[head--] = NULL; -++ r->consumer_tail = r->consumer_head; -++ -++ /* -++ * Go over entries in batch, start moving head back and copy entries. -++ * Stop when we run into previously unconsumed entries. -++ */ -++ while (n) { -++ head = r->consumer_head - 1; -++ if (head < 0) -++ head = r->size - 1; -++ if (r->queue[head]) { -++ /* This batch entry will have to be destroyed. */ -++ goto done; -++ } -++ r->queue[head] = batch[--n]; -++ r->consumer_tail = head; -++ /* matching READ_ONCE in __ptr_ring_empty for lockless tests */ -++ WRITE_ONCE(r->consumer_head, head); -++ } -++ -++done: -++ /* Destroy all entries left in the batch. */ -++ while (n) -++ destroy(batch[--n]); -++ spin_unlock(&r->producer_lock); -++ spin_unlock_irqrestore(&r->consumer_lock, flags); -++} -++ -++static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, -++ int size, gfp_t gfp, -++ void (*destroy)(void *)) -++{ -++ int producer = 0; -++ void **old; -++ void *ptr; -++ -++ while ((ptr = __ptr_ring_consume(r))) -++ if (producer < size) -++ queue[producer++] = ptr; -++ else if (destroy) -++ destroy(ptr); -++ -++ if (producer >= size) -++ producer = 0; -++ __ptr_ring_set_size(r, size); -++ r->producer = producer; -++ r->consumer_head = 0; -++ r->consumer_tail = 0; -++ old = r->queue; -++ r->queue = queue; -++ -++ return old; -++} -++ -++/* -++ * Note: producer lock is nested within consumer lock, so if you -++ * resize you must make sure all uses nest correctly. -++ * In particular if you consume ring in interrupt or BH context, you must -++ * disable interrupts/BH when doing so. -++ */ -++static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, -++ void (*destroy)(void *)) -++{ -++ unsigned long flags; -++ void **queue = __ptr_ring_init_queue_alloc(size, gfp); -++ void **old; -++ -++ if (!queue) -++ return -ENOMEM; -++ -++ spin_lock_irqsave(&(r)->consumer_lock, flags); -++ spin_lock(&(r)->producer_lock); -++ -++ old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy); -++ -++ spin_unlock(&(r)->producer_lock); -++ spin_unlock_irqrestore(&(r)->consumer_lock, flags); -++ -++ kvfree(old); -++ -++ return 0; -++} -++ -++/* -++ * Note: producer lock is nested within consumer lock, so if you -++ * resize you must make sure all uses nest correctly. -++ * In particular if you consume ring in interrupt or BH context, you must -++ * disable interrupts/BH when doing so. -++ */ -++static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, -++ unsigned int nrings, -++ int size, -++ gfp_t gfp, void (*destroy)(void *)) -++{ -++ unsigned long flags; -++ void ***queues; -++ int i; -++ -++ queues = kmalloc_array(nrings, sizeof(*queues), gfp); -++ if (!queues) -++ goto noqueues; -++ -++ for (i = 0; i < nrings; ++i) { -++ queues[i] = __ptr_ring_init_queue_alloc(size, gfp); -++ if (!queues[i]) -++ goto nomem; -++ } -++ -++ for (i = 0; i < nrings; ++i) { -++ spin_lock_irqsave(&(rings[i])->consumer_lock, flags); -++ spin_lock(&(rings[i])->producer_lock); -++ queues[i] = __ptr_ring_swap_queue(rings[i], queues[i], -++ size, gfp, destroy); -++ spin_unlock(&(rings[i])->producer_lock); -++ spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags); -++ } -++ -++ for (i = 0; i < nrings; ++i) -++ kvfree(queues[i]); -++ -++ kfree(queues); -++ -++ return 0; -++ -++nomem: -++ while (--i >= 0) -++ kvfree(queues[i]); -++ -++ kfree(queues); -++ -++noqueues: -++ return -ENOMEM; -++} -++ -++static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) -++{ -++ void *ptr; -++ -++ if (destroy) -++ while ((ptr = ptr_ring_consume(r))) -++ destroy(ptr); -++ kvfree(r->queue); -++} -++ -++#endif /* _LINUX_PTR_RING_H */ -+--- a/include/linux/skbuff.h -++++ b/include/linux/skbuff.h -+@@ -37,6 +37,7 @@ -+ #include -+ #include -+ #include -++#include -+ #include -+ -+ /* A. Checksumming of received packets by device. -+@@ -592,13 +593,23 @@ struct sk_buff { -+ */ -+ kmemcheck_bitfield_begin(flags1); -+ __u16 queue_mapping; -++ -++/* if you move cloned around you also must adapt those constants */ -++#ifdef __BIG_ENDIAN_BITFIELD -++#define CLONED_MASK (1 << 7) -++#else -++#define CLONED_MASK 1 -++#endif -++#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset) -++ -++ __u8 __cloned_offset[0]; -+ __u8 cloned:1, -+ nohdr:1, -+ fclone:2, -+ peeked:1, -+ head_frag:1, -+- xmit_more:1; -+- /* one bit hole */ -++ xmit_more:1, -++ __unused:1; /* one bit hole */ -+ kmemcheck_bitfield_end(flags1); -+ -+ /* fields enclosed in headers_start/headers_end are copied -+@@ -639,6 +650,14 @@ struct sk_buff { -+ __u8 csum_level:2; -+ __u8 csum_bad:1; -+ -++#ifdef __BIG_ENDIAN_BITFIELD -++#define PKT_VLAN_PRESENT_BIT 7 -++#else -++#define PKT_VLAN_PRESENT_BIT 0 -++#endif -++#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset) -++ __u8 __pkt_vlan_present_offset[0]; -++ __u8 vlan_present:1; -+ #ifdef CONFIG_IPV6_NDISC_NODETYPE -+ __u8 ndisc_nodetype:2; -+ #endif -+@@ -647,7 +666,7 @@ struct sk_buff { -+ __u8 remcsum_offload:1; -+ __u8 gro_skip:1; -+ __u8 fast_forwarded:1; -+- /* 1 or 3 bit hole */ -++ /* 0 or 2 bit hole */ -+ -+ #ifdef CONFIG_NET_SCHED -+ __u16 tc_index; /* traffic control index */ -+@@ -805,6 +824,15 @@ static inline struct rtable *skb_rtable( -+ return (struct rtable *)skb_dst(skb); -+ } -+ -++/* For mangling skb->pkt_type from user space side from applications -++ * such as nft, tc, etc, we only allow a conservative subset of -++ * possible pkt_types to be set. -++*/ -++static inline bool skb_pkt_type_ok(u32 ptype) -++{ -++ return ptype <= PACKET_OTHERHOST; -++} -++ -+ void kfree_skb(struct sk_buff *skb); -+ void kfree_skb_list(struct sk_buff *segs); -+ void skb_tx_error(struct sk_buff *skb); -+@@ -2127,6 +2155,11 @@ static inline unsigned char *skb_mac_hea -+ return skb->head + skb->mac_header; -+ } -+ -++static inline u32 skb_mac_header_len(const struct sk_buff *skb) -++{ -++ return skb->network_header - skb->mac_header; -++} -++ -+ static inline int skb_mac_header_was_set(const struct sk_buff *skb) -+ { -+ return skb->mac_header != (typeof(skb->mac_header))~0U; -+@@ -2256,7 +2289,7 @@ static inline int pskb_network_may_pull( -+ -+ int ___pskb_trim(struct sk_buff *skb, unsigned int len); -+ -+-static inline void __skb_trim(struct sk_buff *skb, unsigned int len) -++static inline void __skb_set_length(struct sk_buff *skb, unsigned int len) -+ { -+ if (unlikely(skb_is_nonlinear(skb))) { -+ WARN_ON(1); -+@@ -2266,6 +2299,11 @@ static inline void __skb_trim(struct sk_ -+ skb_set_tail_pointer(skb, len); -+ } -+ -++static inline void __skb_trim(struct sk_buff *skb, unsigned int len) -++{ -++ __skb_set_length(skb, len); -++} -++ -+ void skb_trim(struct sk_buff *skb, unsigned int len); -+ -+ static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) -+@@ -2318,6 +2356,20 @@ static inline struct sk_buff *skb_reduce -+ return skb; -+ } -+ -++static inline int __skb_grow(struct sk_buff *skb, unsigned int len) -++{ -++ unsigned int diff = len - skb->len; -++ -++ if (skb_tailroom(skb) < diff) { -++ int ret = pskb_expand_head(skb, 0, diff - skb_tailroom(skb), -++ GFP_ATOMIC); -++ if (ret) -++ return ret; -++ } -++ __skb_set_length(skb, len); -++ return 0; -++} -++ -+ /** -+ * skb_orphan - orphan a buffer -+ * @skb: buffer to orphan -+@@ -2818,6 +2870,18 @@ static inline int skb_linearize_cow(stru -+ __skb_linearize(skb) : 0; -+ } -+ -++static __always_inline void -++__skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, -++ unsigned int off) -++{ -++ if (skb->ip_summed == CHECKSUM_COMPLETE) -++ skb->csum = csum_block_sub(skb->csum, -++ csum_partial(start, len, 0), off); -++ else if (skb->ip_summed == CHECKSUM_PARTIAL && -++ skb_checksum_start_offset(skb) < 0) -++ skb->ip_summed = CHECKSUM_NONE; -++} -++ -+ /** -+ * skb_postpull_rcsum - update checksum for received skb after pull -+ * @skb: buffer to update -+@@ -2828,36 +2892,38 @@ static inline int skb_linearize_cow(stru -+ * update the CHECKSUM_COMPLETE checksum, or set ip_summed to -+ * CHECKSUM_NONE so that it can be recomputed from scratch. -+ */ -+- -+ static inline void skb_postpull_rcsum(struct sk_buff *skb, -+ const void *start, unsigned int len) -+ { -+- if (skb->ip_summed == CHECKSUM_COMPLETE) -+- skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); -+- else if (skb->ip_summed == CHECKSUM_PARTIAL && -+- skb_checksum_start_offset(skb) < 0) -+- skb->ip_summed = CHECKSUM_NONE; -++ __skb_postpull_rcsum(skb, start, len, 0); -+ } -+ -+-unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); -++static __always_inline void -++__skb_postpush_rcsum(struct sk_buff *skb, const void *start, unsigned int len, -++ unsigned int off) -++{ -++ if (skb->ip_summed == CHECKSUM_COMPLETE) -++ skb->csum = csum_block_add(skb->csum, -++ csum_partial(start, len, 0), off); -++} -+ -++/** -++ * skb_postpush_rcsum - update checksum for received skb after push -++ * @skb: buffer to update -++ * @start: start of data after push -++ * @len: length of data pushed -++ * -++ * After doing a push on a received packet, you need to call this to -++ * update the CHECKSUM_COMPLETE checksum. -++ */ -+ static inline void skb_postpush_rcsum(struct sk_buff *skb, -+ const void *start, unsigned int len) -+ { -+- /* For performing the reverse operation to skb_postpull_rcsum(), -+- * we can instead of ... -+- * -+- * skb->csum = csum_add(skb->csum, csum_partial(start, len, 0)); -+- * -+- * ... just use this equivalent version here to save a few -+- * instructions. Feeding csum of 0 in csum_partial() and later -+- * on adding skb->csum is equivalent to feed skb->csum in the -+- * first place. -+- */ -+- if (skb->ip_summed == CHECKSUM_COMPLETE) -+- skb->csum = csum_partial(start, len, skb->csum); -++ __skb_postpush_rcsum(skb, start, len, 0); -+ } -+ -++unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); -++ -+ /** -+ * skb_push_rcsum - push skb and update receive checksum -+ * @skb: buffer to update -+@@ -2901,6 +2967,21 @@ static inline int pskb_trim_rcsum(struct -+ #define skb_rb_next(skb) rb_to_skb(rb_next(&(skb)->rbnode)) -+ #define skb_rb_prev(skb) rb_to_skb(rb_prev(&(skb)->rbnode)) -+ -++static inline int __skb_trim_rcsum(struct sk_buff *skb, unsigned int len) -++{ -++ if (skb->ip_summed == CHECKSUM_COMPLETE) -++ skb->ip_summed = CHECKSUM_NONE; -++ __skb_trim(skb, len); -++ return 0; -++} -++ -++static inline int __skb_grow_rcsum(struct sk_buff *skb, unsigned int len) -++{ -++ if (skb->ip_summed == CHECKSUM_COMPLETE) -++ skb->ip_summed = CHECKSUM_NONE; -++ return __skb_grow(skb, len); -++} -++ -+ #define skb_queue_walk(queue, skb) \ -+ for (skb = (queue)->next; \ -+ skb != (struct sk_buff *)(queue); \ -+@@ -3662,6 +3743,13 @@ static inline bool skb_is_gso_v6(const s -+ return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; -+ } -+ -++static inline void skb_gso_reset(struct sk_buff *skb) -++{ -++ skb_shinfo(skb)->gso_size = 0; -++ skb_shinfo(skb)->gso_segs = 0; -++ skb_shinfo(skb)->gso_type = 0; -++} -++ -+ void __skb_warn_lro_forwarding(const struct sk_buff *skb); -+ -+ static inline bool skb_warn_if_lro(const struct sk_buff *skb) -+--- a/include/linux/if_arp.h -++++ b/include/linux/if_arp.h -+@@ -44,4 +44,21 @@ static inline int arp_hdr_len(struct net -+ return sizeof(struct arphdr) + (dev->addr_len + sizeof(u32)) * 2; -+ } -+ } -++ -++static inline bool dev_is_mac_header_xmit(const struct net_device *dev) -++{ -++ switch (dev->type) { -++ case ARPHRD_TUNNEL: -++ case ARPHRD_TUNNEL6: -++ case ARPHRD_SIT: -++ case ARPHRD_IPGRE: -++ case ARPHRD_VOID: -++ case ARPHRD_NONE: -++ case ARPHRD_RAWIP: -++ return false; -++ default: -++ return true; -++ } -++} -++ -+ #endif /* _LINUX_IF_ARP_H */ -+--- a/include/linux/if_vlan.h -++++ b/include/linux/if_vlan.h -+@@ -66,7 +66,6 @@ static inline struct vlan_ethhdr *vlan_e -+ #define VLAN_PRIO_MASK 0xe000 /* Priority Code Point */ -+ #define VLAN_PRIO_SHIFT 13 -+ #define VLAN_CFI_MASK 0x1000 /* Canonical Format Indicator */ -+-#define VLAN_TAG_PRESENT VLAN_CFI_MASK -+ #define VLAN_VID_MASK 0x0fff /* VLAN Identifier */ -+ #define VLAN_N_VID 4096 -+ -+@@ -78,8 +77,8 @@ static inline bool is_vlan_dev(struct ne -+ return dev->priv_flags & IFF_802_1Q_VLAN; -+ } -+ -+-#define skb_vlan_tag_present(__skb) ((__skb)->vlan_tci & VLAN_TAG_PRESENT) -+-#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci & ~VLAN_TAG_PRESENT) -++#define skb_vlan_tag_present(__skb) ((__skb)->vlan_present) -++#define skb_vlan_tag_get(__skb) ((__skb)->vlan_tci) -+ #define skb_vlan_tag_get_id(__skb) ((__skb)->vlan_tci & VLAN_VID_MASK) -+ -+ /** -+@@ -376,6 +375,31 @@ static inline struct sk_buff *vlan_inser -+ return skb; -+ } -+ -++/** -++ * __vlan_hwaccel_clear_tag - clear hardware accelerated VLAN info -++ * @skb: skbuff to clear -++ * -++ * Clears the VLAN information from @skb -++ */ -++static inline void __vlan_hwaccel_clear_tag(struct sk_buff *skb) -++{ -++ skb->vlan_present = 0; -++} -++ -++/** -++ * __vlan_hwaccel_copy_tag - copy hardware accelerated VLAN info from another skb -++ * @dst: skbuff to copy to -++ * @src: skbuff to copy from -++ * -++ * Copies VLAN information from @src to @dst (for branchless code) -++ */ -++static inline void __vlan_hwaccel_copy_tag(struct sk_buff *dst, const struct sk_buff *src) -++{ -++ dst->vlan_present = src->vlan_present; -++ dst->vlan_proto = src->vlan_proto; -++ dst->vlan_tci = src->vlan_tci; -++} -++ -+ /* -+ * __vlan_hwaccel_push_inside - pushes vlan tag to the payload -+ * @skb: skbuff to tag -+@@ -390,7 +414,7 @@ static inline struct sk_buff *__vlan_hwa -+ skb = vlan_insert_tag_set_proto(skb, skb->vlan_proto, -+ skb_vlan_tag_get(skb)); -+ if (likely(skb)) -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ return skb; -+ } -+ /* -+@@ -422,7 +446,8 @@ static inline void __vlan_hwaccel_put_ta -+ __be16 vlan_proto, u16 vlan_tci) -+ { -+ skb->vlan_proto = vlan_proto; -+- skb->vlan_tci = VLAN_TAG_PRESENT | vlan_tci; -++ skb->vlan_tci = vlan_tci; -++ skb->vlan_present = 1; -+ } -+ -+ /** -+--- a/include/net/checksum.h -++++ b/include/net/checksum.h -+@@ -120,6 +120,11 @@ static inline __wsum csum_partial_ext(co -+ -+ #define CSUM_MANGLED_0 ((__force __sum16)0xffff) -+ -++static inline void csum_replace_by_diff(__sum16 *sum, __wsum diff) -++{ -++ *sum = csum_fold(csum_add(diff, ~csum_unfold(*sum))); -++} -++ -+ static inline void csum_replace4(__sum16 *sum, __be32 from, __be32 to) -+ { -+ __wsum tmp = csum_sub(~csum_unfold(*sum), (__force __wsum)from); -+--- a/lib/test_bpf.c -++++ b/lib/test_bpf.c -+@@ -38,6 +38,7 @@ -+ #define SKB_HASH 0x1234aaab -+ #define SKB_QUEUE_MAP 123 -+ #define SKB_VLAN_TCI 0xffff -++#define SKB_VLAN_PRESENT 1 -+ #define SKB_DEV_IFINDEX 577 -+ #define SKB_DEV_TYPE 588 -+ -+@@ -691,8 +692,8 @@ static struct bpf_test tests[] = { -+ CLASSIC, -+ { }, -+ { -+- { 1, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT }, -+- { 10, SKB_VLAN_TCI & ~VLAN_TAG_PRESENT } -++ { 1, SKB_VLAN_TCI }, -++ { 10, SKB_VLAN_TCI } -+ }, -+ }, -+ { -+@@ -705,8 +706,8 @@ static struct bpf_test tests[] = { -+ CLASSIC, -+ { }, -+ { -+- { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, -+- { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } -++ { 1, SKB_VLAN_PRESENT }, -++ { 10, SKB_VLAN_PRESENT } -+ }, -+ }, -+ { -+@@ -4432,8 +4433,8 @@ static struct bpf_test tests[] = { -+ CLASSIC, -+ { }, -+ { -+- { 1, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) }, -+- { 10, !!(SKB_VLAN_TCI & VLAN_TAG_PRESENT) } -++ { 1, SKB_VLAN_PRESENT }, -++ { 10, SKB_VLAN_PRESENT } -+ }, -+ .fill_helper = bpf_fill_maxinsns6, -+ }, -+@@ -5144,6 +5145,7 @@ static struct sk_buff *populate_skb(char -+ skb->hash = SKB_HASH; -+ skb->queue_mapping = SKB_QUEUE_MAP; -+ skb->vlan_tci = SKB_VLAN_TCI; -++ skb->vlan_present = SKB_VLAN_PRESENT; -+ skb->dev = &dev; -+ skb->dev->ifindex = SKB_DEV_IFINDEX; -+ skb->dev->type = SKB_DEV_TYPE; -+--- a/include/linux/netdevice.h -++++ b/include/linux/netdevice.h -+@@ -3171,6 +3171,21 @@ int __dev_forward_skb(struct net_device -+ int dev_forward_skb(struct net_device *dev, struct sk_buff *skb); -+ bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb); -+ -++static __always_inline int ____dev_forward_skb(struct net_device *dev, -++ struct sk_buff *skb) -++{ -++ if (skb_orphan_frags(skb, GFP_ATOMIC) || -++ unlikely(!is_skb_forwardable(dev, skb))) { -++ atomic_long_inc(&dev->rx_dropped); -++ kfree_skb(skb); -++ return NET_RX_DROP; -++ } -++ -++ skb_scrub_packet(skb, true); -++ skb->priority = 0; -++ return 0; -++} -++ -+ extern int netdev_budget; -+ -+ /* Called by rtnetlink.c:rtnl_unlock() */ -+--- a/net/openvswitch/actions.c -++++ b/net/openvswitch/actions.c -+@@ -246,7 +246,7 @@ static int push_vlan(struct sk_buff *skb -+ else -+ key->eth.tci = vlan->vlan_tci; -+ return skb_vlan_push(skb, vlan->vlan_tpid, -+- ntohs(vlan->vlan_tci) & ~VLAN_TAG_PRESENT); -++ ntohs(vlan->vlan_tci) & ~VLAN_CFI_MASK); -+ } -+ -+ /* 'src' is already properly masked. */ -+--- a/net/openvswitch/flow.c -++++ b/net/openvswitch/flow.c -+@@ -318,7 +318,7 @@ static int parse_vlan(struct sk_buff *sk -+ return -ENOMEM; -+ -+ qp = (struct qtag_prefix *) skb->data; -+- key->eth.tci = qp->tci | htons(VLAN_TAG_PRESENT); -++ key->eth.tci = qp->tci | htons(VLAN_CFI_MASK); -+ __skb_pull(skb, sizeof(struct qtag_prefix)); -+ -+ return 0; -+--- a/net/openvswitch/flow.h -++++ b/net/openvswitch/flow.h -+@@ -69,7 +69,7 @@ struct sw_flow_key { -+ struct { -+ u8 src[ETH_ALEN]; /* Ethernet source address. */ -+ u8 dst[ETH_ALEN]; /* Ethernet destination address. */ -+- __be16 tci; /* 0 if no VLAN, VLAN_TAG_PRESENT set otherwise. */ -++ __be16 tci; /* 0 if no VLAN, VLAN_CFI_MASK set otherwise. */ -+ __be16 type; /* Ethernet frame type. */ -+ } eth; -+ union { -+--- a/net/openvswitch/flow_netlink.c -++++ b/net/openvswitch/flow_netlink.c -+@@ -925,11 +925,11 @@ static int ovs_key_from_nlattrs(struct n -+ __be16 tci; -+ -+ tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]); -+- if (!(tci & htons(VLAN_TAG_PRESENT))) { -++ if (!(tci & htons(VLAN_CFI_MASK))) { -+ if (is_mask) -+- OVS_NLERR(log, "VLAN TCI mask does not have exact match for VLAN_TAG_PRESENT bit."); -++ OVS_NLERR(log, "VLAN TCI mask does not have exact match for VLAN_CFI_MASK bit."); -+ else -+- OVS_NLERR(log, "VLAN TCI does not have VLAN_TAG_PRESENT bit set."); -++ OVS_NLERR(log, "VLAN TCI does not have VLAN_CFI_MASK bit set."); -+ -+ return -EINVAL; -+ } -+@@ -1209,7 +1209,7 @@ int ovs_nla_get_match(struct net *net, s -+ key_attrs &= ~(1 << OVS_KEY_ATTR_ENCAP); -+ encap_valid = true; -+ -+- if (tci & htons(VLAN_TAG_PRESENT)) { -++ if (tci & htons(VLAN_CFI_MASK)) { -+ err = parse_flow_nlattrs(encap, a, &key_attrs, log); -+ if (err) -+ return err; -+@@ -1297,7 +1297,7 @@ int ovs_nla_get_match(struct net *net, s -+ if (a[OVS_KEY_ATTR_VLAN]) -+ tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]); -+ -+- if (!(tci & htons(VLAN_TAG_PRESENT))) { -++ if (!(tci & htons(VLAN_CFI_MASK))) { -+ OVS_NLERR(log, "VLAN tag present bit must have an exact match (tci_mask=%x).", -+ ntohs(tci)); -+ err = -EINVAL; -+@@ -2272,7 +2272,7 @@ static int __ovs_nla_copy_actions(struct -+ vlan = nla_data(a); -+ if (vlan->vlan_tpid != htons(ETH_P_8021Q)) -+ return -EINVAL; -+- if (!(vlan->vlan_tci & htons(VLAN_TAG_PRESENT))) -++ if (!(vlan->vlan_tci & htons(VLAN_CFI_MASK))) -+ return -EINVAL; -+ vlan_tci = vlan->vlan_tci; -+ break; -+@@ -2288,7 +2288,7 @@ static int __ovs_nla_copy_actions(struct -+ /* Prohibit push MPLS other than to a white list -+ * for packets that have a known tag order. -+ */ -+- if (vlan_tci & htons(VLAN_TAG_PRESENT) || -++ if (vlan_tci & htons(VLAN_CFI_MASK) || -+ (eth_type != htons(ETH_P_IP) && -+ eth_type != htons(ETH_P_IPV6) && -+ eth_type != htons(ETH_P_ARP) && -+@@ -2300,7 +2300,7 @@ static int __ovs_nla_copy_actions(struct -+ } -+ -+ case OVS_ACTION_ATTR_POP_MPLS: -+- if (vlan_tci & htons(VLAN_TAG_PRESENT) || -++ if (vlan_tci & htons(VLAN_CFI_MASK) || -+ !eth_p_mpls(eth_type)) -+ return -EINVAL; -+ -+--- a/net/sched/act_bpf.c -++++ b/net/sched/act_bpf.c -+@@ -220,7 +220,7 @@ static int tcf_bpf_init_from_efd(struct -+ -+ bpf_fd = nla_get_u32(tb[TCA_ACT_BPF_FD]); -+ -+- fp = bpf_prog_get(bpf_fd); -++ fp = bpf_prog_get_type_dev(bpf_fd, BPF_PROG_TYPE_SCHED_ACT, false); -+ if (IS_ERR(fp)) -+ return PTR_ERR(fp); -+ -+--- a/net/sched/cls_bpf.c -++++ b/net/sched/cls_bpf.c -+@@ -267,7 +267,7 @@ static int cls_bpf_prog_from_efd(struct -+ -+ bpf_fd = nla_get_u32(tb[TCA_BPF_FD]); -+ -+- fp = bpf_prog_get(bpf_fd); -++ fp = bpf_prog_get_type_dev(bpf_fd, BPF_PROG_TYPE_SCHED_CLS, false); -+ if (IS_ERR(fp)) -+ return PTR_ERR(fp); -+ -+--- a/net/8021q/vlan_core.c -++++ b/net/8021q/vlan_core.c -+@@ -50,7 +50,7 @@ bool vlan_do_receive(struct sk_buff **sk -+ } -+ -+ skb->priority = vlan_get_ingress_priority(vlan_dev, skb->vlan_tci); -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ -+ rx_stats = this_cpu_ptr(vlan_dev_priv(vlan_dev)->vlan_pcpu_stats); -+ -+--- a/net/ipv4/ip_tunnel_core.c -++++ b/net/ipv4/ip_tunnel_core.c -+@@ -128,7 +128,7 @@ int iptunnel_pull_header(struct sk_buff -+ secpath_reset(skb); -+ skb_clear_hash_if_not_l4(skb); -+ skb_dst_drop(skb); -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ skb_set_queue_mapping(skb, 0); -+ skb->pkt_type = PACKET_HOST; -+ -+--- a/net/bridge/br_netfilter_hooks.c -++++ b/net/bridge/br_netfilter_hooks.c -+@@ -673,10 +673,8 @@ static int br_nf_push_frag_xmit(struct n -+ return 0; -+ } -+ -+- if (data->vlan_tci) { -+- skb->vlan_tci = data->vlan_tci; -+- skb->vlan_proto = data->vlan_proto; -+- } -++ if (data->vlan_proto) -++ __vlan_hwaccel_put_tag(skb, data->vlan_proto, data->vlan_tci); -+ -+ skb_copy_to_linear_data_offset(skb, -data->size, data->mac, data->size); -+ __skb_push(skb, data->encap_size); -+@@ -740,8 +738,13 @@ static int br_nf_dev_queue_xmit(struct n -+ -+ data = this_cpu_ptr(&brnf_frag_data_storage); -+ -+- data->vlan_tci = skb->vlan_tci; -+- data->vlan_proto = skb->vlan_proto; -++ if (skb_vlan_tag_present(skb)) { -++ data->vlan_tci = skb->vlan_tci; -++ data->vlan_proto = skb->vlan_proto; -++ } else { -++ data->vlan_proto = 0; -++ } -++ -+ data->encap_size = nf_bridge_encap_header_len(skb); -+ data->size = ETH_HLEN + data->encap_size; -+ -+--- a/net/bridge/br_private.h -++++ b/net/bridge/br_private.h -+@@ -751,7 +751,7 @@ static inline int br_vlan_get_tag(const -+ int err = 0; -+ -+ if (skb_vlan_tag_present(skb)) { -+- *vid = skb_vlan_tag_get(skb) & VLAN_VID_MASK; -++ *vid = skb_vlan_tag_get_id(skb); -+ } else { -+ *vid = 0; -+ err = -EINVAL; -+--- a/net/bridge/br_vlan.c -++++ b/net/bridge/br_vlan.c -+@@ -354,7 +354,7 @@ struct sk_buff *br_handle_vlan(struct ne -+ } -+ } -+ if (v->flags & BRIDGE_VLAN_INFO_UNTAGGED) -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ -+ out: -+ return skb; -+@@ -420,8 +420,8 @@ static bool __allowed_ingress(struct net -+ __vlan_hwaccel_put_tag(skb, proto, pvid); -+ else -+ /* Priority-tagged Frame. -+- * At this point, We know that skb->vlan_tci had -+- * VLAN_TAG_PRESENT bit and its VID field was 0x000. -++ * At this point, we know that skb->vlan_tci VID -++ * field was 0. -+ * We update only VID field and preserve PCP field. -+ */ -+ skb->vlan_tci |= pvid; -+--- a/net/core/dev.c -++++ b/net/core/dev.c -+@@ -3988,7 +3988,7 @@ ncls: -+ * and set skb->priority like in vlan_do_receive() -+ * For the time being, just ignore Priority Code Point -+ */ -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ } -+ -+ type = skb->protocol; -+@@ -4211,7 +4211,9 @@ static void gro_list_prepare(struct napi -+ } -+ -+ diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; -+- diffs |= p->vlan_tci ^ skb->vlan_tci; -++ diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); -++ if (skb_vlan_tag_present(p)) -++ diffs |= p->vlan_tci ^ skb->vlan_tci; -+ diffs |= skb_metadata_dst_cmp(p, skb); -+ if (maclen == ETH_HLEN) -+ diffs |= compare_ether_header(skb_mac_header(p), -+@@ -4452,7 +4454,7 @@ static void napi_reuse_skb(struct napi_s -+ __skb_pull(skb, skb_headlen(skb)); -+ /* restore the reserve we had after netdev_alloc_skb_ip_align() */ -+ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ skb->dev = napi->dev; -+ skb->skb_iif = 0; -+ skb->encapsulation = 0; -+--- a/net/core/skbuff.c -++++ b/net/core/skbuff.c -+@@ -4491,7 +4491,7 @@ int skb_vlan_pop(struct sk_buff *skb) -+ int err; -+ -+ if (likely(skb_vlan_tag_present(skb))) { -+- skb->vlan_tci = 0; -++ __vlan_hwaccel_clear_tag(skb); -+ } else { -+ if (unlikely((skb->protocol != htons(ETH_P_8021Q) && -+ skb->protocol != htons(ETH_P_8021AD)) || -+--- a/include/net/sock.h -++++ b/include/net/sock.h -+@@ -1484,6 +1484,14 @@ do { \ -+ lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \ -+ } while (0) -+ -++#ifdef CONFIG_LOCKDEP -++static inline bool lockdep_sock_is_held(struct sock *sk) -++{ -++ return lockdep_is_held(&sk->sk_lock) || -++ lockdep_is_held(&sk->sk_lock.slock); -++} -++#endif -++ -+ void lock_sock_nested(struct sock *sk, int subclass); -+ -+ static inline void lock_sock(struct sock *sk) diff --git a/toolchain/kernel-headers/Makefile b/toolchain/kernel-headers/Makefile index c33f26d46d..06236b5a47 100644 --- a/toolchain/kernel-headers/Makefile diff --git a/patches/0051-ipq807x-add-hfcl-ion4xe-i.patch b/patches/0051-ipq807x-add-hfcl-ion4xe-i.patch deleted file mode 100644 index 80c778dc9..000000000 --- a/patches/0051-ipq807x-add-hfcl-ion4xe-i.patch +++ /dev/null @@ -1,992 +0,0 @@ -From a0e1c6853dabbc3bae674e1644f2de67d565fadb Mon Sep 17 00:00:00 2001 -From: John Crispin -Date: Thu, 18 Nov 2021 10:44:40 +0100 -Subject: [PATCH] ipq807x: add hfcl ion4xe/i - -Signed-off-by: John Crispin ---- - .../ipq807x/base-files/etc/board.d/01_leds | 5 + - .../ipq807x/base-files/etc/board.d/02_network | 5 + - .../etc/hotplug.d/firmware/10-ath11k-caldata | 2 + - .../base-files/lib/upgrade/platform.sh | 4 + - target/linux/ipq807x/config-4.4 | 2 +- - .../arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts | 18 + - .../arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts | 18 + - .../boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi | 478 ++++++++++++++++++ - .../dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts | 155 ++++++ - .../dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts | 156 ++++++ - target/linux/ipq807x/image/ipq60xx.mk | 18 + - 11 files changed, 860 insertions(+), 1 deletion(-) - create mode 100644 target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts - create mode 100644 target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts - create mode 100644 target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi - create mode 100644 target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts - create mode 100644 target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts - -diff --git a/target/linux/ipq807x/base-files/etc/board.d/01_leds b/target/linux/ipq807x/base-files/etc/board.d/01_leds -index 1f1797b0c6..ab19fd0532 100755 ---- a/target/linux/ipq807x/base-files/etc/board.d/01_leds -+++ b/target/linux/ipq807x/base-files/etc/board.d/01_leds -@@ -31,6 +31,11 @@ edgecore,eap102) - ucidef_set_led_wlan "wlan5g" "WLAN5G" "green:wifi5" "phy0tpt" - ucidef_set_led_wlan "wlan2g" "WLAN2G" "green:wifi2" "phy1tpt" - ;; -+hfcl,ion4xi|\ -+hfcl,ion4xe) -+ ucidef_set_led_wlan "wlan5g" "WLAN5G" "blue:wifi5" "phy0tpt" -+ ucidef_set_led_wlan "wlan2g" "WLAN2G" "blue:wifi2" "phy1tpt" -+ ;; - esac - - board_config_flush -diff --git a/target/linux/ipq807x/base-files/etc/board.d/02_network b/target/linux/ipq807x/base-files/etc/board.d/02_network -index 9d7dd4e61e..89db0cc5a4 100755 ---- a/target/linux/ipq807x/base-files/etc/board.d/02_network -+++ b/target/linux/ipq807x/base-files/etc/board.d/02_network -@@ -12,6 +12,11 @@ qcom_setup_interfaces() - ucidef_add_switch_attr "switch0" "reset" "false" - - case $board in -+ hfcl,ion4xi|\ -+ hfcl,ion4xe) -+ ucidef_set_interface_wan "eth0 eth1" -+ ucidef_set_interface_lan "" -+ ;; - cig,wf188|\ - tplink,ex227|\ - tplink,ex447) -diff --git a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -index 1788908ab0..ba7991759e 100755 ---- a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -+++ b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -@@ -57,6 +57,8 @@ case "$FIRMWARE" in - cig,wf188|\ - cig,wf188n|\ - edgecore,eap101|\ -+ hfcl,ion4xi|\ -+ hfcl,ion4xe|\ - wallys,dr6018|\ - qcom,ipq6018-cp01|\ - xiaomi,ax1800) -diff --git a/target/linux/ipq807x/base-files/lib/upgrade/platform.sh b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -index 407db099f6..ef8cce8614 100755 ---- a/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -+++ b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -@@ -28,6 +28,8 @@ platform_check_image() { - edgecore,eap101|\ - edgecore,eap102|\ - edgecore,eap106|\ -+ hfcl,ion4xi|\ -+ hfcl,ion4xe|\ - tplink,ex227|\ - tplink,ex447|\ - qcom,ipq6018-cp01|\ -@@ -53,6 +55,8 @@ platform_do_upgrade() { - cig,wf188n|\ - cig,wf194c|\ - cig,wf194c4|\ -+ hfcl,ion4xi|\ -+ hfcl,ion4xe|\ - qcom,ipq6018-cp01|\ - qcom,ipq807x-hk01|\ - qcom,ipq807x-hk14|\ -diff --git a/target/linux/ipq807x/config-4.4 b/target/linux/ipq807x/config-4.4 -index 1b4b95b968..cef3f951b6 100644 ---- a/target/linux/ipq807x/config-4.4 -+++ b/target/linux/ipq807x/config-4.4 -@@ -638,7 +638,7 @@ CONFIG_SND_PCM=y - CONFIG_SND_SOC=y - # CONFIG_SND_SOC_APQ8016_SBC is not set - CONFIG_SND_SOC_I2C_AND_SPI=y --CONFIG_SND_SOC_IPQ=y -+# CONFIG_SND_SOC_IPQ is not set - # CONFIG_SND_SOC_IPQ806X_LPAIF is not set - # CONFIG_SND_SOC_IPQ806X_PCM_RAW is not set - CONFIG_SND_SOC_IPQ_ADSS=y -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts -new file mode 100644 -index 0000000000..5a44b9a3e7 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xe.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts -new file mode 100644 -index 0000000000..8edd1d817a ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-hfcl-ion4xi.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi -new file mode 100644 -index 0000000000..7c67cf7ade ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-cp01-hfcl.dtsi -@@ -0,0 +1,478 @@ -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "qcom-ipq6018.dtsi" -+#include -+#include -+ -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ compatible = "qcom,ipq6018-cp01", "qcom,ipq6018"; -+ interrupt-parent = <&intc>; -+ qcom,msm-id = <0x192 0x0>, <0x193 0x0>; -+ -+ aliases { -+ serial0 = &blsp1_uart3; -+ serial1 = &blsp1_uart2; -+ sdhc2 = &sdhc_2; -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet0 = "/soc/dp1"; -+ ethernet1 = "/soc/dp2"; -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -+#ifdef __IPQ_MEM_PROFILE_256_MB__ -+ bootargs-append = " swiotlb=1"; -+#else -+ bootargs-append = " swiotlb=1 coherent_pool=2M"; -+#endif -+ }; -+ -+}; -+ -+&tlmm { -+ pinctrl-0 = <&sd_ldo_pins &hfcl_gpio>; -+ pinctrl-names = "default"; -+ -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio44", "gpio45"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ sd_ldo_pins: sd_ldo_pins { -+ mux { -+ pins = "gpio66"; -+ function = "gpio"; -+ drive-strength = <2>; -+ bias-disable; -+ output-low; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ sd_pins: sd_pins { -+ mux { -+ pins = "gpio62"; -+ function = "sd_card"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ extcon_usb_pins: extcon_usb_pins { -+ mux { -+ pins = "gpio26"; -+ function = "gpio"; -+ drive-strength = <2>; -+ bias-pull-down; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ reset_button { -+ pins = "gpio53"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio64"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_2 { -+ pins = "gpio75"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ mux_3 { -+ pins = "gpio77"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ }; -+ -+ leds_pins: leds_pins { -+ led_5g { -+ pins = "gpio60"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_2g { -+ pins = "gpio61"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ hfcl_gpio: hfcl_gpio { -+ mux_0 { -+ pins = "gpio25"; -+ function = "gpio"; -+ drive-strength = <8>; -+ output-low; -+ }; -+ mux_1 { -+ pins = "gpio29"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_2 { -+ pins = "gpio33"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_3 { -+ pins = "gpio34"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_4 { -+ pins = "gpio35"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_5 { -+ pins = "gpio59"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_6 { -+ pins = "gpio67"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_7 { -+ pins = "gpio70"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ mux_8 { -+ pins = "gpio79"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ uniphy_pins: uniphy_pinmux { -+ mux { -+ pins = "gpio63"; -+ function = "rx0"; -+ bias-disable; -+ }; -+ sfp_tx { -+ pins = "gpio48"; -+ function = "gpio"; -+ driver-strength = <8>; -+ bias-pull-down; -+ output-low; -+ }; -+ }; -+ -+ hsuart_pins: hsuart_pins { -+ mux { -+ pins = "gpio71", "gpio72"; -+ function = "blsp1_uart"; -+ drive-strength = <8>; -+ bias-disable; -+ }; -+ }; -+ -+}; -+ -+&soc { -+ extcon_usb: extcon_usb { -+ pinctrl-0 = <&extcon_usb_pins>; -+ pinctrl-names = "default"; -+ id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; -+ status = "disabled"; -+ }; -+ -+ mdio: mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 77 0>; -+ status = "ok"; -+ phy0: ethernet-phy@0 { -+ reg = <4>; -+ }; -+ phy1: ethernet-phy@1 { -+ reg = <30>; -+ }; -+ }; -+ -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <4>; -+ reg = <0x3a001600 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <4>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <5>; -+ reg = <0x3a003000 0x3fff>; -+ qcom,mactype = <1>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <30>; -+ phy-mode = "sgmii"; -+ }; -+ -+ nss-macsec1 { -+ compatible = "qcom,nss-macsec"; -+ phy_addr = <30>; -+ phy_access_mode = <0>; -+ mdiobus = <&mdio>; -+ }; -+ -+ ess-switch@3a000000 { -+ pinctrl-0 = <&uniphy_pins>; -+ pinctrl-names = "default"; -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x10>; /* lan port bitmap */ -+ switch_wan_bmp = <0x20>; /* wan port bitmap */ -+ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -+ switch_mac_mode = <0xf>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0x14>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -+ qcom,port_phyinfo { -+ port@0 { -+ port_id = <4>; -+ phy_address = <4>; -+ }; -+ port@2 { -+ port_id = <5>; -+ phy_address = <30>; -+ phy_i2c_address = <30>; -+ phy-i2c-mode; -+ port_mac_sel = "GMAC_PORT"; -+ }; -+ }; -+ }; -+ -+ gpio_keys { -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ pinctrl-names = "default"; -+ -+ reset { -+ label = "reset"; -+ linux,code = ; -+ gpios = <&tlmm 53 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ }; -+ }; -+ -+ i2c_2: i2c@78b9000 { -+ compatible = "qcom,i2c-qup-v2.2.1"; -+ #address-cells = <1>; -+ #size-cells = <0>; -+ reg = <0x78b9000 0x600>; -+ interrupts = ; -+ clocks = <&gcc GCC_BLSP1_AHB_CLK>, -+ <&gcc GCC_BLSP1_QUP5_I2C_APPS_CLK>; -+ clock-names = "iface", "core"; -+ clock-frequency = <400000>; -+ dmas = <&blsp_dma 21>, <&blsp_dma 20>; -+ dma-names = "rx", "tx"; -+ status = "disabled"; -+ }; -+}; -+ -+&blsp1_uart3 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+ -+&blsp1_uart2 { -+ pinctrl-0 = <&hsuart_pins>; -+ pinctrl-names = "default"; -+ dmas = <&blsp_dma 2>, -+ <&blsp_dma 3>; -+ dma-names = "tx", "rx"; -+ status = "ok"; -+}; -+ -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_1 { -+ status = "ok"; -+}; -+ -+&usb2 { -+ status = "ok"; -+}; -+ -+&usb3 { -+ status = "ok"; -+}; -+ -+&nss_crypto { -+ status = "ok"; -+}; -+ -+&pcie_phy { -+ status = "disabled"; -+}; -+ -+&pcie0 { -+ status = "disabled"; -+}; -+ -+&qpic_lcd { -+ status = "disabled"; -+}; -+ -+&qpic_lcd_panel { -+ status = "disabled"; -+}; -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts -new file mode 100644 -index 0000000000..a64ec25a8a ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xe.dts -@@ -0,0 +1,155 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "qcom-ipq6018-cp01-hfcl.dtsi" -+#include "qcom-ipq6018-rpm-regulator.dtsi" -+#include "qcom-ipq6018-cpr-regulator.dtsi" -+#include "qcom-ipq6018-cp-cpu.dtsi" -+ -+/ { -+ model = "HFCL ION4Xe"; -+ compatible = "hfcl,ion4xe", "qcom,ipq6018-cp01", "qcom,ipq6018"; -+ -+ /* -+ * +=========+==============+========================+ -+ * | | | | -+ * | Region | Start Offset | Size | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | | | | -+ * | | | | -+ * | | | | -+ * | | | | -+ * | Linux | 0x41000000 | 139MB | -+ * | | | | -+ * | | | | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | TZ App | 0x49B00000 | 6MB | -+ * +--------+--------------+-------------------------+ -+ * -+ * From the available 145 MB for Linux in the first 256 MB, -+ * we are reserving 6 MB for TZAPP. -+ * -+ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+ * for memory layout. -+ */ -+ -+/* TZAPP is enabled in default memory profile only */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+ reserved-memory { -+ tzapp:tzapp@49B00000 { /* TZAPPS */ -+ no-map; -+ reg = <0x0 0x49B00000 0x0 0x00600000>; -+ }; -+ }; -+#endif -+}; -+ -+&tlmm { -+ i2c_1_pins: i2c_1_pins { -+ mux { -+ pins = "gpio42", "gpio43"; -+ function = "blsp2_i2c"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ i2c_2_pins: i2c_2_pins { -+ mux { -+ pins = "gpio55", "gpio56"; -+ function = "blsp4_i2c"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+&i2c_1 { -+ pinctrl-0 = <&i2c_1_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+ -+ lm75@48 { -+ compatible = "lm75"; -+ reg = <0x48>; -+ status = "okay"; -+ }; -+}; -+ -+&i2c_2 { -+ pinctrl-0 = <&i2c_2_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&sdhc_2 { -+ pinctrl-0 = <&sd_pins>; -+ pinctrl-names = "default"; -+ cd-gpios = <&tlmm 62 1>; -+ sd-ldo-gpios = <&tlmm 66 0>; -+ vqmmc-supply = <&ipq6018_l2_corner>; -+ status = "ok"; -+}; -+ -+&soc { -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ -+ led@60 { -+ label = "blue:wifi5"; -+ gpios = <&tlmm 60 GPIO_ACTIVE_LOW>; -+ linux,default-trigger = "led_5g"; -+ default-state = "off"; -+ }; -+ led@61 { -+ label = "blue:wifi2"; -+ gpios = <&tlmm 61 GPIO_ACTIVE_LOW>; -+ linux,default-trigger = "led_2g"; -+ default-state = "off"; -+ }; -+ }; -+}; -+ -+&tlmm { -+ leds_pins: leds_pins { -+ led_5g { -+ pins = "gpio60"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_2g { -+ pins = "gpio61"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+/* TZAPP is enabled in default memory profile only */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+&qseecom { -+ mem-start = <0x49B00000>; -+ mem-size = <0x600000>; -+ status = "ok"; -+}; -+#endif -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts -new file mode 100644 -index 0000000000..88b609f37e ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-hfcl-ion4xi.dts -@@ -0,0 +1,156 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "qcom-ipq6018-cp01-hfcl.dtsi" -+#include "qcom-ipq6018-rpm-regulator.dtsi" -+#include "qcom-ipq6018-cpr-regulator.dtsi" -+#include "qcom-ipq6018-cp-cpu.dtsi" -+ -+/ { -+ model = "HFCL ION4Xi"; -+ compatible = "hfcl,ion4xi", "qcom,ipq6018-cp01", "qcom,ipq6018"; -+ -+ /* -+ * +=========+==============+========================+ -+ * | | | | -+ * | Region | Start Offset | Size | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | | | | -+ * | | | | -+ * | | | | -+ * | | | | -+ * | Linux | 0x41000000 | 139MB | -+ * | | | | -+ * | | | | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | TZ App | 0x49B00000 | 6MB | -+ * +--------+--------------+-------------------------+ -+ * -+ * From the available 145 MB for Linux in the first 256 MB, -+ * we are reserving 6 MB for TZAPP. -+ * -+ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+ * for memory layout. -+ */ -+ -+/* TZAPP is enabled in default memory profile only */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+ reserved-memory { -+ tzapp:tzapp@49B00000 { /* TZAPPS */ -+ no-map; -+ reg = <0x0 0x49B00000 0x0 0x00600000>; -+ }; -+ }; -+#endif -+}; -+ -+&tlmm { -+ i2c_1_pins: i2c_1_pins { -+ mux { -+ pins = "gpio42", "gpio43"; -+ function = "blsp2_i2c"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ i2c_2_pins: i2c_2_pins { -+ mux { -+ pins = "gpio55", "gpio56"; -+ function = "blsp4_i2c"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+&i2c_1 { -+ pinctrl-0 = <&i2c_1_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+ -+ lm75@48 { -+ compatible = "lm75"; -+ reg = <0x48>; -+ status = "okay"; -+ }; -+}; -+ -+&i2c_2 { -+ pinctrl-0 = <&i2c_2_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&sdhc_2 { -+ pinctrl-0 = <&sd_pins>; -+ pinctrl-names = "default"; -+ cd-gpios = <&tlmm 62 1>; -+ sd-ldo-gpios = <&tlmm 66 0>; -+ vqmmc-supply = <&ipq6018_l2_corner>; -+ status = "ok"; -+}; -+ -+&soc { -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ -+ led@60 { -+ label = "blue:wifi5"; -+ gpios = <&tlmm 32 GPIO_ACTIVE_LOW>; -+ linux,default-trigger = "led_5g"; -+ default-state = "off"; -+ }; -+ led@61 { -+ label = "blue:wifi2"; -+ gpios = <&tlmm 31 GPIO_ACTIVE_LOW>; -+ linux,default-trigger = "led_2g"; -+ default-state = "off"; -+ }; -+ }; -+}; -+ -+&tlmm { -+ leds_pins: leds_pins { -+ led_5g { -+ pins = "gpio32"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_2g { -+ pins = "gpio31"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+ -+/* TZAPP is enabled in default memory profile only */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+&qseecom { -+ mem-start = <0x49B00000>; -+ mem-size = <0x600000>; -+ status = "ok"; -+}; -+#endif -diff --git a/target/linux/ipq807x/image/ipq60xx.mk b/target/linux/ipq807x/image/ipq60xx.mk -index e94ab22785..30ec2160bc 100644 ---- a/target/linux/ipq807x/image/ipq60xx.mk -+++ b/target/linux/ipq807x/image/ipq60xx.mk -@@ -20,6 +20,24 @@ define Device/cig_wf188n - endef - TARGET_DEVICES += cig_wf188n - -+define Device/hfcl_ion4xe -+ DEVICE_TITLE := HFCL ION4Xe -+ DEVICE_DTS := qcom-ipq6018-hfcl-ion4xe -+ DEVICE_DTS_CONFIG := config@cp01-c1 -+ SUPPORTED_DEVICES := hfcl,ion4xe -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 uboot-envtools -+endef -+TARGET_DEVICES += hfcl_ion4xe -+ -+define Device/hfcl_ion4xi -+ DEVICE_TITLE := HFCL ION4Xi -+ DEVICE_DTS := qcom-ipq6018-hfcl-ion4xi -+ DEVICE_DTS_CONFIG := config@cp01-c1 -+ SUPPORTED_DEVICES := hfcl,ion4xi -+ DEVICE_PACKAGES := ath11k-wifi-qcom-ipq6018 uboot-envtools -+endef -+TARGET_DEVICES += hfcl_ion4xi -+ - define Device/edgecore_eap101 - DEVICE_TITLE := EdgeCore EAP101 - DEVICE_DTS := qcom-ipq6018-edgecore-eap101 --- -2.25.1 - diff --git a/patches/0053-ipq807x-add-support-fro-wallytech-DR6018-v4.patch b/patches/0053-ipq807x-add-support-fro-wallytech-DR6018-v4.patch deleted file mode 100644 index 737b9491b..000000000 --- a/patches/0053-ipq807x-add-support-fro-wallytech-DR6018-v4.patch +++ /dev/null @@ -1,622 +0,0 @@ -From 1eb2dea202bbb22ba34130ce9802e72c1c03cf7a Mon Sep 17 00:00:00 2001 -From: John Crispin -Date: Mon, 29 Nov 2021 10:43:53 +0100 -Subject: [PATCH] ipq807x: add support fro wallytech DR6018-v4 - -Signed-off-by: John Crispin ---- - .../ipq807x/base-files/etc/board.d/01_leds | 1 + - .../ipq807x/base-files/etc/board.d/02_network | 4 + - .../etc/hotplug.d/firmware/10-ath11k-caldata | 1 + - .../base-files/lib/upgrade/platform.sh | 2 + - .../dts/qcom-ipq6018-wallys-dr6018-v4.dts | 18 + - .../qcom/qcom-ipq6018-wallys-dr6018-v4.dts | 492 ++++++++++++++++++ - target/linux/ipq807x/image/ipq60xx.mk | 9 + - 7 files changed, 527 insertions(+) - create mode 100644 target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts - create mode 100755 target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts - -diff --git a/target/linux/ipq807x/base-files/etc/board.d/01_leds b/target/linux/ipq807x/base-files/etc/board.d/01_leds -index ab19fd0532..d47a7e55ed 100755 ---- a/target/linux/ipq807x/base-files/etc/board.d/01_leds -+++ b/target/linux/ipq807x/base-files/etc/board.d/01_leds -@@ -9,6 +9,7 @@ board=$(board_name) - case "$board" in - sercomm,wallaby|\ - wallys,dr6018|\ -+wallys,dr6018-v4|\ - cig,wf188n|\ - cig,wf194c|\ - cig,wf194c4) -diff --git a/target/linux/ipq807x/base-files/etc/board.d/02_network b/target/linux/ipq807x/base-files/etc/board.d/02_network -index 89db0cc5a4..7c81bb50e7 100755 ---- a/target/linux/ipq807x/base-files/etc/board.d/02_network -+++ b/target/linux/ipq807x/base-files/etc/board.d/02_network -@@ -50,6 +50,10 @@ qcom_setup_interfaces() - ucidef_set_interface_lan "eth0 eth1 eth2 eth3" - ucidef_set_interface_wan "eth4" - ;; -+ wallys,dr6018-v4) -+ ucidef_set_interface_lan "eth1 eth2 eth3 eth4" -+ ucidef_set_interface_wan "eth0" -+ ;; - esac - } - -diff --git a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -index ba7991759e..7f081a6b5f 100755 ---- a/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -+++ b/target/linux/ipq807x/base-files/etc/hotplug.d/firmware/10-ath11k-caldata -@@ -60,6 +60,7 @@ case "$FIRMWARE" in - hfcl,ion4xi|\ - hfcl,ion4xe|\ - wallys,dr6018|\ -+ wallys,dr6018-v4|\ - qcom,ipq6018-cp01|\ - xiaomi,ax1800) - caldata_extract "0:ART" 0x1000 0x20000 -diff --git a/target/linux/ipq807x/base-files/lib/upgrade/platform.sh b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -index ef8cce8614..50ba925a48 100755 ---- a/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -+++ b/target/linux/ipq807x/base-files/lib/upgrade/platform.sh -@@ -25,6 +25,7 @@ platform_check_image() { - cig,wf194c|\ - cig,wf194c4|\ - wallys,dr6018|\ -+ wallys,dr6018-v4|\ - edgecore,eap101|\ - edgecore,eap102|\ - edgecore,eap106|\ -@@ -62,6 +63,7 @@ platform_do_upgrade() { - qcom,ipq807x-hk14|\ - qcom,ipq5018-mp03.3|\ - wallys,dr6018|\ -+ wallys,dr6018-v4|\ - tplink,ex447|\ - tplink,ex227) - nand_upgrade_tar "$1" -diff --git a/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts -new file mode 100644 -index 0000000000..470e437e41 ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm/boot/dts/qcom-ipq6018-wallys-dr6018-v4.dts -@@ -0,0 +1,18 @@ -+/* -+ * Copyright (c) 2020, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "../../../arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts" -+#include "qcom-ipq6018.dtsi" -diff --git a/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts -new file mode 100755 -index 0000000000..a00c106adc ---- /dev/null -+++ b/target/linux/ipq807x/files/arch/arm64/boot/dts/qcom/qcom-ipq6018-wallys-dr6018-v4.dts -@@ -0,0 +1,492 @@ -+/dts-v1/; -+/* -+ * Copyright (c) 2019, The Linux Foundation. All rights reserved. -+ * -+ * Permission to use, copy, modify, and/or distribute this software for any -+ * purpose with or without fee is hereby granted, provided that the above -+ * copyright notice and this permission notice appear in all copies. -+ * -+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -+ */ -+ -+#include "qcom-ipq6018.dtsi" -+#include "qcom-ipq6018-rpm-regulator.dtsi" -+#include "qcom-ipq6018-cpr-regulator.dtsi" -+#include "qcom-ipq6018-cp-cpu.dtsi" -+#include -+#include -+ -+/ { -+ #address-cells = <0x2>; -+ #size-cells = <0x2>; -+ model = "Wallys DR6018 V4"; -+ compatible = "wallys,dr6018-v4", "qcom,ipq6018-cp01", "qcom,ipq6018"; -+ interrupt-parent = <&intc>; -+ -+ aliases { -+ serial0 = &blsp1_uart3; -+ serial1 = &blsp1_uart2; -+ -+ /* -+ * Aliases as required by u-boot -+ * to patch MAC addresses -+ */ -+ ethernet0 = "/soc/dp1"; -+ ethernet1 = "/soc/dp2"; -+ ethernet2 = "/soc/dp3"; -+ ethernet3 = "/soc/dp4"; -+ ethernet4 = "/soc/dp5"; -+ -+ sdhc2 = "/soc/sdhci_sd@7804000"; -+ -+ led-boot = &led_power; -+ led-failsafe = &led_power; -+ led-running = &led_power; -+ led-upgrade = &led_power; -+ }; -+ -+ chosen { -+ bootargs = "console=ttyMSM0,115200,n8 rw init=/init"; -+ bootargs-append = " console=ttyMSM0,115200,n8 swiotlb=1 coherent_pool=2M"; -+ }; -+ -+ /* -+ * +=========+==============+========================+ -+ * | | | | -+ * | Region | Start Offset | Size | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | | | | -+ * | | | | -+ * | | | | -+ * | | | | -+ * | Linux | 0x41000000 | 139MB | -+ * | | | | -+ * | | | | -+ * | | | | -+ * +--------+--------------+-------------------------+ -+ * | TZ App | 0x49B00000 | 6MB | -+ * +--------+--------------+-------------------------+ -+ * -+ * From the available 145 MB for Linux in the first 256 MB, -+ * we are reserving 6 MB for TZAPP. -+ * -+ * Refer arch/arm64/boot/dts/qcom/qcom-ipq6018-memory.dtsi -+ * for memory layout. -+ */ -+ -+/* TZAPP is enabled only in default memory profile */ -+#if !defined(__IPQ_MEM_PROFILE_256_MB__) && !defined(__IPQ_MEM_PROFILE_512_MB__) -+ reserved-memory { -+ tzapp:tzapp@49B00000 { /* TZAPPS */ -+ no-map; -+ reg = <0x0 0x49B00000 0x0 0x00600000>; -+ }; -+ }; -+#endif -+}; -+ -+&tlmm { -+ uart_pins: uart_pins { -+ mux { -+ pins = "gpio44", "gpio45"; -+ function = "blsp2_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ sd_pins: sd_pins { -+ mux { -+ pins = "gpio62"; -+ function = "sd_card"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ spi_0_pins: spi_0_pins { -+ mux { -+ pins = "gpio38", "gpio39", "gpio40", "gpio41"; -+ function = "blsp0_spi"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ qpic_pins: qpic_pins { -+ data_0 { -+ pins = "gpio15"; -+ function = "qpic_pad0"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_1 { -+ pins = "gpio12"; -+ function = "qpic_pad1"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_2 { -+ pins = "gpio13"; -+ function = "qpic_pad2"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_3 { -+ pins = "gpio14"; -+ function = "qpic_pad3"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_4 { -+ pins = "gpio5"; -+ function = "qpic_pad4"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_5 { -+ pins = "gpio6"; -+ function = "qpic_pad5"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_6 { -+ pins = "gpio7"; -+ function = "qpic_pad6"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ data_7 { -+ pins = "gpio8"; -+ function = "qpic_pad7"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ qpic_pad { -+ pins = "gpio1", "gpio3", "gpio4", -+ "gpio10", "gpio11", "gpio17"; -+ function = "qpic_pad"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ -+ extcon_usb_pins: extcon_usb_pins { -+ mux { -+ pins = "gpio26"; -+ function = "gpio"; -+ drive-strength = <2>; -+ bias-pull-down; -+ }; -+ }; -+ -+ button_pins: button_pins { -+ wps_button { -+ pins = "gpio19"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ }; -+ -+ mdio_pins: mdio_pinmux { -+ mux_0 { -+ pins = "gpio64"; -+ function = "mdc"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_1 { -+ pins = "gpio65"; -+ function = "mdio"; -+ drive-strength = <8>; -+ bias-pull-up; -+ }; -+ mux_2 { -+ pins = "gpio75"; -+ function = "gpio"; -+ bias-pull-up; -+ }; -+ }; -+ -+ leds_pins: leds_pins { -+ led_pwr { -+ pins = "gpio74"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_5g { -+ pins = "gpio35"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ led_2g { -+ pins = "gpio37"; -+ function = "gpio"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+ uart2_pins: uart2_pins { -+ mux { -+ pins = "gpio57", "gpio58"; -+ function = "blsp4_uart"; -+ drive-strength = <8>; -+ bias-pull-down; -+ }; -+ }; -+}; -+ -+&soc { -+ extcon_usb: extcon_usb { -+ pinctrl-0 = <&extcon_usb_pins>; -+ pinctrl-names = "default"; -+ id-gpio = <&tlmm 26 GPIO_ACTIVE_LOW>; -+ status = "ok"; -+ }; -+ -+ mdio: mdio@90000 { -+ pinctrl-0 = <&mdio_pins>; -+ pinctrl-names = "default"; -+ phy-reset-gpio = <&tlmm 75 0 &tlmm 77 1>; -+ status = "ok"; -+ ethernet-phy@3 { -+ reg = <0x03>; -+ }; -+ -+ ethernet-phy@4 { -+ reg = <0x18>; -+ }; -+ -+ ethernet-phy@1 { -+ reg = <0x01>; -+ }; -+ -+ ethernet-phy@2 { -+ reg = <0x02>; -+ }; -+ -+ ethernet-phy@0 { -+ reg = <0x00>; -+ }; -+ }; -+ -+ dp1 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <1>; -+ reg = <0x3a001000 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp2 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <2>; -+ reg = <0x3a001200 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <1>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp3 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <3>; -+ reg = <0x3a001400 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <2>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp4 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <4>; -+ reg = <0x3a001600 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <3>; -+ phy-mode = "sgmii"; -+ }; -+ -+ dp5 { -+ device_type = "network"; -+ compatible = "qcom,nss-dp"; -+ qcom,id = <5>; -+ reg = <0x3a001800 0x200>; -+ qcom,mactype = <0>; -+ local-mac-address = [000000000000]; -+ qcom,link-poll = <1>; -+ qcom,phy-mdio-addr = <0x18>; -+ phy-mode = "sgmii"; -+ }; -+ -+ ess-switch@3a000000 { -+ switch_cpu_bmp = <0x1>; /* cpu port bitmap */ -+ switch_lan_bmp = <0x3c>; /* lan port bitmap */ -+ switch_wan_bmp = <0x2>; /* wan port bitmap */ -+ switch_inner_bmp = <0xc0>; /*inner port bitmap*/ -+ switch_mac_mode = <0x0>; /* mac mode for uniphy instance0*/ -+ switch_mac_mode1 = <0xf>; /* mac mode for uniphy instance1*/ -+ switch_mac_mode2 = <0xff>; /* mac mode for uniphy instance2*/ -+ qcom,port_phyinfo { -+ port@1 { -+ phy_address = <0x01>; -+ port_id = <0x02>; -+ }; -+ -+ port@0 { -+ phy_address = <0x00>; -+ port_id = <0x01>; -+ }; -+ -+ port@2 { -+ phy_address = <0x02>; -+ port_id = <0x03>; -+ }; -+ -+ port@3 { -+ phy_address = <0x03>; -+ port_id = <0x04>; -+ }; -+ -+ port@4 { -+ phy_address = <0x18>; -+ port_id = <0x05>; -+ port_mac_sel = "QGMAC_PORT"; -+ }; -+ }; -+ }; -+ -+ gpio_keys { -+ compatible = "gpio-keys"; -+ pinctrl-0 = <&button_pins>; -+ pinctrl-names = "default"; -+ -+ reset { -+ label = "reset"; -+ linux,code = ; -+ gpios = <&tlmm 19 GPIO_ACTIVE_LOW>; -+ linux,input-type = <1>; -+ debounce-interval = <60>; -+ }; -+ }; -+ -+ leds { -+ compatible = "gpio-leds"; -+ pinctrl-0 = <&leds_pins>; -+ pinctrl-names = "default"; -+ -+ led@25 { -+ label = "green:wifi5"; -+ gpios = <&tlmm 35 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:5g"; -+ default-state = "off"; -+ }; -+ led@24 { -+ label = "green:wifi2"; -+ gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:2g"; -+ default-state = "off"; -+ }; -+ led_power: led@16 { -+ label = "green:led_pwr"; -+ gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>; -+ linux,default-trigger = "green:power"; -+ default-state = "off"; -+ }; -+ }; -+}; -+ -+&blsp1_uart3 { -+ pinctrl-0 = <&uart_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&spi_0 { -+ pinctrl-0 = <&spi_0_pins>; -+ pinctrl-names = "default"; -+ cs-select = <0>; -+ status = "ok"; -+ -+ m25p80@0 { -+ #address-cells = <1>; -+ #size-cells = <1>; -+ reg = <0>; -+ compatible = "n25q128a11"; -+ linux,modalias = "m25p80", "n25q128a11"; -+ spi-max-frequency = <50000000>; -+ use-default-sizes; -+ }; -+}; -+ -+&blsp1_uart2 { -+ pinctrl-0 = <&uart2_pins>; -+ pinctrl-names = "default"; -+ dmas = <&blsp_dma 2>, -+ <&blsp_dma 3>; -+ dma-names = "tx", "rx"; -+ status = "ok"; -+}; -+&qpic_bam { -+ status = "ok"; -+}; -+ -+&nand { -+ pinctrl-0 = <&qpic_pins>; -+ pinctrl-names = "default"; -+ status = "ok"; -+}; -+ -+&ssphy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_0 { -+ status = "ok"; -+}; -+ -+&qusb_phy_1 { -+ status = "ok"; -+}; -+ -+&usb2 { -+ status = "ok"; -+}; -+ -+&usb3 { -+ status = "ok"; -+}; -+ -+&nss_crypto { -+ status = "ok"; -+}; -+ -+&sdhc_2 { -+ pinctrl-0 = <&sd_pins>; -+ pinctrl-names = "default"; -+ cd-gpios = <&tlmm 62 1>; -+ sd-ldo-gpios = <&tlmm 66 0>; -+ //vqmmc-supply = <&ipq6018_l2_corner>; -+ status = "ok"; -+}; -diff --git a/target/linux/ipq807x/image/ipq60xx.mk b/target/linux/ipq807x/image/ipq60xx.mk -index 30ec2160bc..4c779435f5 100644 ---- a/target/linux/ipq807x/image/ipq60xx.mk -+++ b/target/linux/ipq807x/image/ipq60xx.mk -@@ -56,6 +56,15 @@ define Device/wallys_dr6018 - endef - TARGET_DEVICES += wallys_dr6018 - -+define Device/wallys_dr6018_v4 -+ DEVICE_TITLE := Wallys DR6018 V4 -+ DEVICE_DTS := qcom-ipq6018-wallys-dr6018-v4 -+ DEVICE_DTS_CONFIG := config@cp01-c4 -+ SUPPORTED_DEVICES := wallys,dr6018-v4 -+ DEVICE_PACKAGES := ath11k-wifi-wallys-dr6018-v4 uboot-envtools -+endef -+TARGET_DEVICES += wallys_dr6018_v4 -+ - define Device/qcom_cp01_c1 - DEVICE_TITLE := Qualcomm Cypress C1 - DEVICE_DTS := qcom-ipq6018-cp01-c1 --- -2.25.1 -

U-}fUDc57aIK~`w2->qKYQ(06LTP`p2oK-vSXwaFV*qOdF -zw0zv8pxxSG&~(YD7V?Djf*1R4*T}ca$LS&))smV($;h!swo7&9vdD{c4l2qtIBAkE -zM0uWxopIRz^wz4j=-~;w!?o$^*2i;R_lCFlaz&X;QL8y>Vju7N#ym0q7F`}`|Apgv -z>}1hIC*9Ga#lN}#g|ZO5L+r@?ngx~0ui`;?q_DSu19Fl*=$2=q>@Hk7tGk%M=qf6n>|kvb06FoT%5lr>Ko4 -zA(!V^h_Y=^b$M!yT9Q1+Qs_JHLgO+?k0v*315^xAj?RaF_d2Z4N>EK-f%~7oUoztS`qzS6 -zUJG6MTG*5N8Lt_wuc@uCG402DG;ffU;}106q})jMkgCfMvUnJa$>9AvV7erZdwQ#X -zfrl^bfOPC3(Hw~V$_a&iO)JYmZxt)EM9VA^nEq5dyncC_YVz+0rATP#&C2b|q6sG* -zUcb&06Uu#R$tkZ*aJ%Z;a^k^t-=lIcx{Z2CA+>yguRO{o>*%vxE2$bJ{2N=DJHKTm -z|C-f$MV+}hL*9Msx<}JB*QuLXjvHBU<#6D@esZp|!bfq#MMkA1x*S)tjMuW(UCTrPu%Zd(>$(3Vy5#s3b1xCYlhpFdl&=6ua-!R@bw7U`|886#f{E0qI6anAoe -zEe+;ybe@vsj+0ppr?S}A>*8^>pRS7}rUBxU(U(tT33~v{7!aFa0Aoy&!|RnmqNAXF -zf~%6VWo`?rk42DPKzSW|`nr4m7s>{&A-Ia_thjwx1A6SWW-x2IeO}Aed`LIj{P2+G -z<&bfH#eIHiA0~wXCbcfm5q2VeC55GGqRd*9v~b!Phu#KMZi8!YU>Zk1_B=TGcqK_%$1BO -zq@(hK3%Tv;UfNUV((RhxIP~?0H -z`N;Vc@{#l1F&_4FvU!y>VlGU-@ZUI>8u0)Sb7k(~n(G1IBe@YDa$&T0wH7FZn?ToMo}*Q$SnfJ^8fViml)`L13-;f);pzvMul%77 -zdw(7FfjW$>nQ)#UWAs-^RGp01T_scShCeX>g7zvY2IG&v<9G|bO3vX8e>ac?f88h$ -za#cSUZ}>wNe(=>gnf@F*7#<-h{ZJW?Wj5Mc!QXoX+qn=vMuOY2_XkwSQaDM=E;N0J=y}h;Md~XU%Z?lTq8G*{bc9EnG_m*O_hYUH)|%^!^Kiyzzqx4U!X>IVMcG*Tkn*wMSn+WyC$TP3 -zoiFZ+#T>;KD`x6kDf!C#v6~(pO}`2Go>{3N^N-C`cu5j}J*^$t7HIo-^cfHGm-{o% -zox9OoRwvap9}7I|flk0)E9NWQI8(;U(oVRrNrSL19`~Q0F;jYUvLx00_1bvLbC$`? -zT0tXei^c&MK3LLbIx~Tg(~o)WE0tZt0XeaDxVho~yzs#Ixv -zMcswtsmzq^%1tpI)XgbQ(SH--_U_yKfB1UyxF)Xk4>&uL3`kfa5KsmjRKOO{R%^8d -z+(COqE4H?_1Us>6t-FANhM*Rd-a=?qTyB%-$fC3)z4wY*?@eM0sL*P(b*c1f+HtEQ -z;*Lw+?-{iB{@%~~dHuAy|8&ZdDD3R6#X&XNwy;vKxU&=(_ -zD9&VEYJ?Va-9ZGWUE-!*YLP{qbDSROZValCGM3w-^pr;F3E#CfK`nj_bwy|%KX%9M -z=i@Ik8gsxVS(~y@RI93>Hx$tuYPC_Nm8Ej#Txg@*Id9!4rFrB|u+ZW!xvRZ{Rfm)V -zMoF~?D9Od(4OH(6%?6`+TdD8EzWe=CdTKq-7Px`E$v1*;8FkQc;2VD8rAbOe;ER|S -zw9kg&frPV#aOUsDi3>#o!Q+@HPZTouq)8HQ=uxgi8p|)mCE;aYF2M)HqtdD7?Ng>2 -zOL`fmJE9hfUJ0KV$Bn*J_Ya7tMqjeHi}@>j)Spe)Gr#C#6EEv68t~) -zt(O?)h$_y&^n|2+JB%doH3$`9A3f6}M0u${l%}MS+)zY({IR!*yt=Fe))57bkx1=*fsek&(IWrALvB_$0mor*pLV**A7b -zic*Sc_IbWRj!lEaKkq~;A0DT33sP+g!xpzYsC&~AFNdT0OU1%JwkCsb(@V5T3Ro}h -zo_1JVxF{$E>l_q{KMGAOnPxC#i+ZI*mD>mh$nx3RiUGw@Tfc#s1pe!MP?R~lB5YmJ -znze&{%>#Fu-ozKcHR{yDHeAYfY73Qr{1e4K^v+TORz%7?^qLv>qV=~XOVyIv%UGJQkpK>AowT=8h_Cnn|Q -zN*_23q37c7qCTUeqJEExIbkR^M!iUHh<-jE9vLe-pQUC0a+!5H=zZ+1g=0NnGO@s3h*zi6RmE48~ix!=5ox)|G) -z68@Ce`lcv)Iw569{R~6U$jn*N8^l+{MTkRTc_8I~Ms{0!33}!{$|(2+e3PFLwfiVo -ze)hu`xi-YY2;w#w7b7q(fJfJgLSUwS-fNC_!2E>7F~<-`Nq)A9RC)n(;}>sST(&uz -zIqJnBd_Ml|q*qY3Zb7;T^6c(;Zq_>gU1IoW4E{A*_*b*}4pw4fXYU~YD#5>Ut>jmz -zKk9w>V_qxyBhU^}R&kV_ekJqm#VO!}DZn`NNy#fd{$X*hB9ii@4aQCHh+0);*qJTA -z7u%MuEsv`N9`JB%Y;7_ok*^g;{9&z7@=acQg+F`5Yt57pkpV9Pd9gpqUZH>_7wra& -zAP5sIM~m?uL=9T-d(^6@!>koB%^?{{p+Xz)l`SqSNgo~cVd_+=NIN8ZdLq!>;k(KJ -zQ7;2!>^tKtJJ=vTz`J`>e%r|u3(m@IiqotJ9M -zE{V!DyFV8-v>VULTtOnJs<*0-NKGMNL5iSrFVC2s`}wcCHE)9#BPpTH|1{^N?oTr0 -znp;xeQtyt4rW6zC;o=%uqZDRN^X6rYxgtL!&6s|z{VcHib3f9%NPwFlJk_ -zqvED*-BezlRMk7f`!Ms9FB_l#J{hxxk|Nn!(1#k!ykaV3K%IDj?AfqIh|M%#Y8=CXN#Y)&Wd%thaajF{k#O{;Hao<@*O!9 -zth-kf)sQ5zs%zEz#Xo-|N=(eALU+h1IjoXiNrIasdhZAI;*B<@OS@^S7*rZ&{qUqM -zboR~wb5#)xb6!cRqTynZ#HFOBLcc#XrQDL5SXxv|mnLaGkTr^1)UB#-&yzy<8(L&< -zRJSW@#G7EB6LE&Vp?Z)675&bGj{WNU*cPdCWrjLOtr%rBY%S_Mraq#s62lWk)bZ>o -zgN20yW3pcQwtAGTmY%HOZem@<;<`YYdFK<#x6QY#x@-d#xadfwYb@poF0kBN_`tdo -z$Zu^j@X7n~U&Drh{~D}d?z0Yo9}Lt5V43!TWm-4q``~div6JI(b=8V-wC~2jFjqqk -z5NrS}WCkU%5n{r@0tTj%n6Y-Vv7#;F)+0he%QQ`BR2&YnvS2U3ZtkWO^$Kb}|M%PV -zib*PV|81>aKkm9vW>X)kyHU-5+5tMS8g=30{h^a;dJ4Nl^H#TcE9Fo1<~|_i&9+{| -ziv~a!0*MqK25*V|E19(&th3J{m|$?FD1rZ~7cO_<@B)Wy%GV_qDgG_cxfW9qv1&6g -zF9%ob{efWS`5Y7i -zJo!<|vNr|)EWc|9gI|%E_lZd^(o{Xd;wxt^YblxH23fDfVr&svGpS@7=>kCZkCg+yenc(0p%5*(nbRhwO$B=LgnKLSN#(gS!Cp -zG>nH64cMpyU^psHe<{s#mbFu#kquOS{VC3@C3hH&W*I -zqH_oLQ~I&L!uSrwpCa&FpG<~U2P!UOFKWR)C-I#gI>Z)RRQ54?yhW=JW7u-d;NcO4 -zE6il91wg=hd{jFPEB!E$A|;A&v1r)1F4`G2EG|@=4ickZ-!?g)&6^vt@o=>-90h-V -zg^ai#>m{~;A|PW4eGfKoR=0v|81a>_ezRXq@QX?;rD9U3!{+nRcxo6dZfogL%0(-q -zss{w4-s7<+hV`wRPD;F8BZz$gg$~}R7}U_rx3SJv4+zkXpE4yIJtXBW3ZE7Rdo4bF -z%fSB=pC;#BlUr((Oh`Twn$_rI_1&VG>0`NIASMya9Zl8-g*~QU)*d<35HYgD*_q)} -zr7LSQAra2FjXe~ca5lahM}l;qIN?x`+L9?!r8r2H{pwZxgxig&)w`t&l#^B3gZhmB -zEZXg$YO4ngU_QJ#BV0j74jT~Zj_IduchBzEB{g_r`cb`b2sX~{7ah|tvU7aMsd^cl -zPxw5f;=xlNSuf-FgpnBDm&EAc6h_7z2qV45t0L>=R6Hkp*ruf7>t$?h*gzeLp+<5< -z0!4!CzA$oYSY0KatKVP3+& -z@k6ywr0udEvznsC2C91lzhONzlFk(y-O{OUpES23?YmP`zmGDu1edB^{nu#TRk}k~ -zYo7RGQUr)#HmWG_c6KK|r1ddDmaK_7=7%FZ#O-YK1Rjt447-RREF3Q1SAZJ&v~#z#bbTHdN&)q1?u&HWGr9~4=Cjn`{)Gp1{!$D4j41z|0V82*WU -zv_-i-%2ZxgW%d=V_ZN*z7Bvli=Eg=bP_#Zqr07T7*g%p4)T;Gr!F7#P1Y>T6gs5JO -zgG(Ekh<*X*4_j~%b=`7*9sjk45IKUM6w94;{8v4X#Bs9a);gx=k(kF=ZmeVKg_qPZ -zD#3DX9lN*Z=~XDfU0TQP5uR*|ieT)1aUEOttQ_Lo;Ma9rZBN=qNPBJ_SJRXBfB}vL -zeqP5__dM3avy0`Yb^LDOQTS)F!BLM-9A9_w_&TZ|_ai}h)Vj=_ph^CU=O2V;=d>cq -z&jXmCeq38m#3V#qw68%#tCw%TZzKRFGJaj3))7<~A>n+>9-<&DJ#Mo>$cPgMBDPT@ -zPkgrK;$M-GZy^zT5V2v+#SJ8)nM52!#QHTC*O7=O67df8nEDjkZ$JKf@;IM*j3R~S -z@5u8iRK9$T@O)TMa6>;r2YAQ29*sSd=2_qJ@qXBT1_G;=P4=^FUI$p?of=R%8PE=4 -zCgH-SkbC`%t|$<|a8wd$=YEvYGIQnUZ5E_fKV(~OzyIl3i#+c!Pj`?NQ}#wXl+_Zw38(Os^|;;EBG -zj+Ru{WK^|52uMxoA_!M{WG$w2B4{zCOQDDL^GQ)!&f@GQb-|dWpw1t0u?!0SuMO)TT27O0c5Mn -zSRpZHD)?}1ZR-9^BI9SmwQgtV{>8OAyEC9h)#BHvJrl|I_10SUS6E(M7ol=h1iCUc -zTpw-iMYnU{{u7>x@D{&~wlk46Di5HgwD3q*OWSp7w{!SLTZ^BkBHELwAa5HXEce2c -z+pY_8Rm9qzQ(T#GB$f%pDlhb}4to`=85qR;<34%>+wyix=V~@j14>d(y__$uRgaRm -zPfDz9yG%-{T`4hCsj<%DYFCQkNU$diP`gWEl5vcvMtyx{z0B3N#SS!43P)Wv@YQg2 -zQ(_k->IuW_kr&?tMTGsA?b|3eo|G6{b}i{8HT9Aha@FOzQKnvE1fXSyFan0$D08k6 -zO~)jWBeoBh=mp{Sx4y2E5?5=(|Nob`K<&(#1X4C*NY7tn>2KG|?a_-}Ze(`w{#twV -zLN-8~9KED&NOH~+?h&rr7;rFTVN6n -z@mb(wbxyH6vt7!?X>KNE}6K#_-K7^jV4V!Mk0XOr9hLWay?JZ&fROQx|r$&i6Txk -zNtvKRilssdC|ZA{ousnrqO7|9ZHh^7;?NU6iJ6a#v8ZFcoSmb!(@`EedgCt}J1+cA -zA3sH3YxmHMi;fkZueC$>&BOh2`USr(^TT~#%Z=t8%>#k_WOx!|xE^CRBpJ5GRN)+Z -zRSa~Dhp8h+G;}5%Q6ImMbT!jWM;rc#F|@^8_(pyFcv8dPF>qtJhZ3l6lM{?l*S|~0 -z#0CaxhnNmO&MNJQ4fN5Dv3~bBEABe!ceGz)fA%{cvhC_e-=ac<$i`)a;mBFIT{n7O -z+(7R-`!~+#t;9&-IQp$W?n}b*ZFzAmGxEQ2Sl8p$3NP4~p0De~1ltn3 -zmV4z{GEyGW(QC;IK3Pac_(hop1Nhfbi)4xBH(?%PG(UBPM7lCXIbyL%EP|$3Llrm& -zQH%6ZIjLe(swhoRy-l0`+&g%Gs&D*Y93XQpgR -zZDPmo5Ihzb;NHn`wA>!CTGVdAf?o@Ih3!hToU@Sa -zuVItrXc|0ImP3VXjE4X3^Myh-TEisCXDOLPIgTnx@-`(7k#e?Qmt~mIR$T~q!s^$U -zD2?SMqtLu=b2r;hW7)NaG$OTt>#MQ+SZv#YbSJpgT%^X*RxG@U&{ -zKBf}f=I{7ED1@7JpDbK6?n5Xu6ZS#u;tn*;OCrmvLM-ppb}Et$(OAAJW(VHF)fT@7 -zmdc};V0`Rje#KSTPvu=D+!E$Lw?L1?@e47@ncs<)+$Ev|a0Q^~96?6suj7&LATMr_ -z7q@Xiu68dJo<9k(ylu2`g~Yg-^GC@aaHl(A7kswU@}{w;Sr|W!B@+!setrQf#T{1m5@-$>zB}mZD;Q9kj`+=gKW>iWw3l -z~551x|%>@-@wm%kR=ICW+004MTEGbqU -zJ>%#2p%T~?Wz_bddNIfJnWeN8tY%l;*W7&!g}-_UHz$x8`Skg@ojAs@vBmt`U1YiA -z95_cp{(KlWoWza-+y-7R$_}F*F@25Z4-9oiZiS%wM$U!6_+&0?(6qjU)C<%2`X=pP -zgBcMWb31BLFt)b0yA~*cVw4>=N;%7CGSv4(&lFq;%I1u&@paP4%2^V^VBk}P199ye -zGx}sSrWUk9{p-L#P?VEfWB=$j_f|OAXm5q%WUXZ0>;lIooS)VOf-8=l)-_lO;H#-! -zPoy_w?ean5MGbdq)!7bUBpm1K&~gGl3EZBM+EK8^NWE3i2&LKdgWMZks6$3qo3cjg -zI&WyMQ=}h2H%;v70UIHj_MnRiFA91i!?iFj14Gc4{z -z;xm1`Q4;>K7ZV3HqAV>%?GpBn5MVAy*c))1f%OPlc^KZPc7mnB$ajPQMG*UAV08>C -z_WXOTF(kY@5WJ`eRhu~e1e^QRW&XfkA+}bC?8hEB>Q1n40J{R)_DkB>pvc&maB9ct -zppe+n@v%z>#umrNRt}{8bvX!58P>z-;HHt<7**$BCJAsCkS9Q8^5_~!hpri$;GglD -z5&BC4{g1VhE3i2VsBf7IAmPX!bX-+z -z{{?INA?X8^$ZL6sq6TaODMmiw-X)^^*^md?=|(y@L$4|^>VmUM?%U}Q6RpNStEw{U -zG7j2xAy!?GNvFX`XROFGT4}9|4o{}TjCAO5X$f}x -zJ^)a1Y^yZsLeL<6L~U{;p|ni*edIYLt&#Hs!)qZv81Tg{53P06aItZDM|cD%I^0Uzar%81YL&6dO63Ux$%(U!X*DO-O{-P65Xuj2+`MNoV-~q+tp~tR^cjCU9j)*WXMZ&?kAFa@F{y2E$JP=!Y5`NxY_5QsY%TiKP_nEPTL~$X8Xi^RNEukU9e8`mvC&cbWUb|4k?qhb -zs;pfK@(fe{UXie=w@Vq}vFp!A3*N$wV@HwAj^vE_`TQ?{SSt6ouqjj0cZbL(zy9ua>O -zs<6ht4W|zDcZS8E86sYy#(}^GTY=pZih$Ocw~DI^HAx`8pAnV-NvjMRYa)1j_%IZ~ -zdMOEJg|;c5KQM2Fx@2PQm>JvMBs-cvl9UrqQGA3@AVqNlp2aYK2K=AD@(XqQh$j2Y -zhz$I*z=_Wbye`z|%~g~iyG2Ov%v?o9@)bR$jimheP0xdoY&i+y;KDBc(RWbO&|aYk3W69?kEs%l8zcX3n3%^hi>~s=9z1S~E03YW+Cjoq% -z6$0erl_cgJx -zcuEU}DUgc8Z6(?Vw-IIRM&b9qN+!JSw*rV6}5pHWZrKYq%5CFzXc -zS^vmRva&)Om>LAN!dkKI4+HnxlRWi^4;HO356~QI^l1?{*ieXb-(FR{YQK20!saos -z7oUvK&&*Xkqb2x{I`whk?4nL*yWzyohN4r3i{}isUkv6em6me`asmZ+9R~`SoR$4n -zZIw|g$H_$dtbqU_#k=sRitNoVfFDL^jsbgRR5~nlshI-?LzD=-u$L{OL7AMsQy(5& -zal}Avur(W~kB*4<>uS|ii&4&rCIj`}t-}T?uT=a{cU66Maf0u#K@!DFXwODnjTmQl -z?ERTthc7%#y;^j(;)sp%YgtSnPS1H8)wM~EgJ%0L-u-mR;dwW}Cxi -zDHA`SDP>9K1WTT6!wxog)J748wb4hc~ -z&rb~{<(+VmHq0Gs*wTF5l3hKBx-ut071iOmyhj4y{I1wFr(YaXGox`IHHl3REZGFHI1J7)pbUu2P2A%Vx~0 -z-VaY-t)AtJL0S5VsY+CpJ7FJmMm%4Lj(_DK)kzp$bJ={z+^8~t1<*5o)04*PN$0Eg -zM7LZ9qP$tN|r4Rxa%2Q<_<66WuV^R*s5Cn)hia{TPf1HJx1 -zokhQ{flotVE|7kYlR(4O5N(X={}0eaPN)wFJjsT+YBuCX*;e+22l{jw6-*ADGA0%_ -zKTor^zym-FIYeL+>C0^kEgOoJcQ0H_@(B-!Z7n&5Va4&I{MjX5a^So{7V1l7pIe-3 -zxKcoGv$Y{?=gnEljTUYNYAz6E5$wZ0Lf~Cq0+&S~Gv3dS@MC5PkX2_L*h2@6d2I}qaTk&A~=ZBh!DgUoVd -zW+epEmI@N>v@oBC2n}*6j9c2%AWz`m&hAX)wD`qCa+fnIQuZgJ2hb|#v?6UZh^9?9(!gn!BsonpA4WPNtK}g)o@C8^Xr=pRDIeNtr~<&BFtQ{cR!^1))c2Hu -zmEQ)&2GX(70s7>jmhd&UU-JMr-5E@fBs6OT;+wk}tWbz7!E0=%@s8^V#y(;R=y}KA -z2)0fWY5T{cclWG648(3LnA^V2b9q%tD{3zd^w0yk5QVc`AP$vXnFBqU{eh`V_$1p_ -zxj!-jAC`=UkJe(`|4R-wGkT~=t`EY2k2;USDtfJ<>Z?eX4OFZ@;5{MEx6=a?9Wn!) -z=eXdmLk7MPhR+c{%`bnTo9EnGFZiXmJX&qrnMXb5e+zDnBihqk(=&aoi&29tlxm4H -zvEZI_d*0mfInBVuWge=dn*$&^NQ!E>`*vn@rqZ6Ns93tHCJB|Z&NY@~;%@QerJW8i -z&Rx630v!|LM`(N-+~DCns)ozRGt!oD2*#m4aU?vWZv55`tQla;;K(m9W^mrI$ntHg -zQ76ie`6<{U&bPJXQRhK(p9-e!c}WY24U93m^Zpg0|7?oxEZ-5_dIUXxOKsVo&!yi3 -z+vC}}DV*&^P-FYt1E-M?!DInpXN0nIL+N>W!i4>sFkyKMrUb79o-C1P481N)ZWS|? -zXxyYh1-{3JC$smaFc!=ME17ODnG4~fItx2H#JUr>(mBlmnAx3ShMh6C>eUeu%s3DP -zqZDWew80^Hjg+7Cgv{>om4sZhLefAfUGPRU21eP~uDMq+j(Ih-JfPl@uE00xB=BQe -zheUr}0v~X@Zejr(PEnv0@OjJCJiaBkbsl#Mq7(KV;%#>ahD0zXDF+`rASfa1;Cvu& -z2aCbg9AJfY>*X*g-av&ud6y^}okItKQDYe5PE4jD5|MC{ur^FNdvgH5Ahr7RYv%tX -z&1|3SU;Qt!wtbeL{@z)f!nrTORtV-d$-zgIQ+_U>w^e#HalZWYu>EwcI>SZ;cKkbG -zwVC@1w>awVSgqO -zlZ1%C=!*XJQk9(x4Ku_MeR7?M`7wl&o1%k@USAFHfX$6^t6MR3UIiNpIWOPmUiA>z -z@!aFa1VcKO60Di0Vz%7ZC{>h(NeeF9z-xf5PW2%hln)iu`X(H@sTRFj%ze?V3HRkT -zcf%bS@Y91d|L}n-iCbYkI>8-haK){uRiFMAGT^LN6;{avIH-QohieL<;K=!m8p-VQ -zwz~(f!K!fd!&>q2Z;SV8x$i={+zR*r_csnIu;v77ON?_1JPWHl3%7a}Zey)?Yk*Bt -zHX9|wZfjd*LHG9j?bE3RgO!4=BALmQqRH*0s0L^);5nyv}@i -z*J>(Z%L&#A^uap{Gdl`XI|_%(C&&!N=83(uL;hJQrlhaPh4)EU3jef4r1?Tf-A9{O4jKh(@q+7Ca_ -zX#HJ>pJ>keyAMCsME3R^?$-Rv*YaHFS0laIp2F6_ZX~V -zRYWUOs~(hT`L+=3Vrt`wjmAf*TFGEbSri8 -z_{hp}N8kv8tGZ9kZfZhq2BD>JBGrWi9n|g4*D4iG0mmq~L^6D8YvV)-cj|s?y5)mB -z_U>IY@0W0%zTc1`lT)3P~obos&6iZQP#r=(2@Iw -zTOfv&M&sxLPuVIg&TmNyJdvH|9tC{#BmO7ZmGC&!6#d)yhY9=_A+DJFwt0oX2x&eT -z3BWylc%O@Dx$c-VR=3Q0wB$m2lza9WEtRwoM6I2DF^qZniP@ZOvWh_Az8hvWHem5t>F=A3mVe8g>>5^%( -zCDY_haUxElYLm5#GcLlX4UY8nlI+G}?yk2+oH?wpQQanwAKNbD&W1kLQyN~OYExo~ -zHC4)@E9K??PEr{~p(!H0rQUZgoxL!LdDXLpKJd@F+Rfzn-zH&>dDc@RCiW)PN66<9 -z@?ChAuijsjYgx1~i67=cp6cA5Jd_XL>}i*^$s5Ivt?DXijha6ZYOR!K)LEy=^NHFE -z;n7i2;$|&qy-Z~BFSPw_1w}ElgAE&EU~f_?-Yd0ie%8La;MQ|6rQ&+s?6I6`5{tNi -zn^*;F#JZ;i(VOJMH7$z7za-VQ9#L2~$-t)o76$i%hl)J#rpi+{8v1b%lcv`n|98@` -zzr!YR=E2iF)npHO+K5IokTVW?8vfxh?{ZZCU939DdLGwm*c-~$nUHLBf?a??7<3cs -z$VJ$)K}@kkr)Gu_&LR*%g9{N804ZP+BJyux1|%U0yJ39?Tn%Dh-7DPdSaJ`X^M&6Q -zS<$J2a+728aLM|35&W^$R9v~`rUP1v7Ih9y@q%KK9X~$w!iAycy)wYuIXcwW+JP*T -zcwuO*&CsFl?jW*!^;yeV|5_EizbLtc;94g{jJTaZHR6M7FQO9h0^^e()Kiy1!59^B -z%phzlz(5=r{E9lCo|`lg{sscC*v?okbgNq~9uY@$8@`V*w8wOIL^aqQ4^G&Bkr_H- -zu4XnIOugcSn4f(FMl<*%6o3}^OVFRjUta3K&&Kh52R<-@q=Y#5qS -zqBiKkVOAfg+Mq9y&zn1Ei}H-j`FX){x8wA;Q=o%M{lW~2m@6M7;}-t`!EuntzVixL -z>*hT>P3H!fjsy(*z1IU>y)PIrVo}ZFC%#oWLsLv+(Ur_IskLI%G2$BPi0YBb-6C9Y -zjH@D1iM(P}LFS~0ky|dq48vfGe3qx*J%H(+KV`z{7HNq3We7Ts3BRV7rd3ZU9Q#p4h -z6l7L6^B|P<51FOZ$M|B=Y#k1meyKd3<$x1CC2Cy@t~>S#ihVPTe-m&0lbCkySE*yZH#fr9Vipg1}iQ -zwwN{v)`p0qOq+y1+=m*fj|oRAYF3-f+*aw*EHIV()x_{>K%81)uqdD>C*8sd*kTRi&x=> -zoce{mB!v|H@3+x6CiKlKR2a^DdiX-7CuN0up&|JS7O(AIOX+fZHFhc14{HFgxNiH) -za;q*IS{)h7St{66{bB3wG`G*x54NU#_u)8|NWUV~az(kG#-UZ25AP -z?aNgz-A7AwI%+Q?4^Gof*tle>bIS2e>MrM7R^5!EEvuj;tlEFeqkG$^o2uH~rJH8b -zO`ocJbBS&qjtxGh!%wN+InD`IT^1E;)XmJ%A=?K9_XuK?Zmx^|Xm3G6H@h<^cU~0y -zXIg{OzFq_8qjowhqC4dvrdk*=p|b`N$>vA;m^m0_9y&Yg#_+6v4ez4gR(Z2tA5PR2 -zxKlw!I;XmZh@O86GSM@vx;Hi-kgPu`BH9YSjysLTdV-anZjWE%rYGz>(AR}u^zRyM -z0QunsjGA?Up*4an2s6K6!%<4!`7lQ96 -zllGXszD0zg(M8X1@uT)MgLNntd&tnayyJ$6o=x=<+G&A?3DSqWh7NVpA6%blr$2DM -zx~gG}Lmv}4{#!`4#g>E1h{TG&7L+vtFl_uf?|PBw_!!5<wd$JvVE@2=kmu)Avua=3dsv^lrR$Z=2FZFQ7&_+KOGe1s-~VzOM?jdySl-v -zHXMxVY=uM6C_`%u)tB7cV;GajginO|pl2{w=dW=ceyTa2t~#k1m8v=UI$CmRs8ttb -z*Yz`1yk^Qw`&kM8KewbOZmg$bgC{f14))AJ;Y7DI6ugvnqGS=D%l;G;O)mglOtWda -ziF*r>%^u{YbMR1o2%ZIzh=A*uvA$_}|A$6X3;_S*!uzyEMns#}O -z^a52$R%3WVT;s)-3cNgH@#sFd;%oriU@K{b=Z%B5xOj9Wn&ZO3D5gkcS-sM>dL^}; -z{UHd-L(qgFP_(b*vu#0GVBYnPf6*t6PEScUrjNy_G17Wiy8&S6lO4CQC_%gKYI@4S -z(fcpLF*My_OI=R2!eK9@_Lh!i$33#+CuHaie2_gHghp5vWcz(N1t-QYL;7vumvwHG -zb)J`X-jH1odd<)t)%jF*+^Ysffjo>OhzeQ%F@e8(mD?3GX{()n!$r^Fmjm^{sCygJ -zJgce|rEbTp$OLV*Er`$-%(ftbYS|i;Vi;VN`l`A|cNdS$-3e5jSqN~uzB -zE!mL!&v}w28cQD;P -z2RXqMw?KudK#yUT9{fVbw>uT$m*V2<|)|sI;Nb+PQ8`y)=f@1YD^jHKK<ZY*qO6Xb+?eI94Q8-gkk@bElhl<_KMSLWBZ;F+U1a7PJCIX -zf|vy5`Zq+WXY)~Xy^MYI7`UKy1Vw!iRwSZgOeu{Ojs!A-Y{iO -z#R25jH6~nS)g{<<@t%q-tXl44NuJD^_G9YjR3q!xQ_PF?Qdh-$uFP~`7~6G8R$Zcy -zI<>{mevGbBor#2hEhUl;gz1ZJh^s>H%ABeny-i4@{+~ol01~}y*S$ogyDH|nGC$_~ -zhT3(nSarjNq&kxH?Vj@3*vCTBS1o(0e%;0s*nbJFPI)o|p6UC`Vakek7EeVm -zA_q*v#>TuA#0P}dp4jgM$I$K^N^*pHG9!9&@bb`2Tj4)hpLT)@+#-W8QCn379hrMo -z-2m5wAX(D%Q1IcmFyFQ?pGdf%9=C6VShdIYQND9Qej^N+%B@>PydTK+{c2Z*#FeQS -zo+R+5N+doX23XE7Ayb)I8Uz+!IO8wyk+GP+pKtqgDNf<4Fx#yBTDnbQc{iU?hS}cD -zuT8?kJNc|{SZyjyQ>k$h5p=GhS3{Mz6kD?LZAX@HK4G?&CADHCoR-fk!fMse655e4 -z9Lv7)eotko#)6ZEQZ6wYD+&ZIW3^ -z%jkSg95(rRLL(CPqbPU;AE+KKwv5WRjmqb|p)>zk_xXdQGxSQ=l~gKlL`Qe~wfeff -zb$jZnE`p!7+O7Gc}Ll0F1qw_c+R_W9sKj==$q -zuos~obMMs)_HZB!W7d~tAQ$|I*=LH@cL}z3saWzeqw<|}e!ZMq9soE5^zRd8n*NOTGM|-U2T7^9m!n5qIT)r0gZ3gj7d4@NU3vJkDjp~JF|cM{yw$CGWIb( -zVONHo8EvR9z{RmP|Sxq1NcLNX90mi3Is_V{c98j;|?uPDABHg!-2hAz@ -zVE7n!Ft8gO=T0V$+PVE34AvPiC{#ohJJKv7!oLoSGLTsUCJ9crzYN% -z;g4Fy6Shcy{-hG1p*`nevDoUI6s-$LCMte`Q*}W(y5J?c5au!PAM)JhRSourEn3eu -z(dKi@TqnF#dPZyXyQ>qZ(khC4kS?S3w8_Wus9gK2 -z`~AY*eUqY{*%5~$J_D}Tv2qHwsMRmZu?4Rei>hP93F+;Ua~q6jyijQ$_`31k)$y(S -z{`N~JL5k+CK9iSZ?$4{2^S_0rev_9REH)1ry*udTdb!P>*Cu8E2sO%xPDc&EPCS;~OT(wBJ#xDesnVeo1a-9cZNm@)`D;qT;*@jAQNu;Vcf_ -z+yly_)p@lzbcb0F2b-i(MyQF(%GXvJM^oHSp#$n9X``uVDbZRJ_=@z>Jgdo{JE^S6 -z!2*K8#GF#rOzjEsr^vX}fp7MTnArzC7PgISK##rS0|&(h2E-;0tC*b^o7OLODvW2) -z^z~x)8U$)(cD5U$QT`a<2bd!$B6%W3rLBMHr2?Z7nhNzA%jw1TU7@hGqmtJH00ZA( -z?aEQg0b5_=e^7qA0kqgRO8V=?O@M4o=W}72&ZYq;gdfGjr2b#I{Aif7^CR)Vj0x7n -zdAYBY)j>aDTWcWoB|dUFyOIEqFh7&^B93$rx8mO35wRoa*z~@!Gqvys(4sOOeW;nM -zvLAh*foJ5S-I@dy1PEdg|4&e`Y?BfY7?SZ>va%5KfJ#u~O-fC4FPzly0t5m7C?xgd -z3hHMp3=uEQ(DsM&-{sgnA#=2Wx!T^$~|LGklV5#_@i5`}U{Uxl+A}WephWA|`7FiZ7A-ZHxzY7e) -zVN8aZ#@C~k>wog{b>x0hz#)jp%qVajXN*f -z9s}~YR#ggU8^XU@;tYsQ^N$Tw>63fWcXNkJsL(PP|Cb7kAoR^!Qy1^c9i9UcG_Xpt -zadg(8z#2(Qa#niart^W?^HaVpw{8=iOAI=-MXtY?HaSNjJ8d -zwuz4)l9Ey9{M5rR|O#YGeaBWh2t)QCb+N6d(h3+OBWwK7`4J|JuPV0<;N&vC+a -zJh*(!-+Gx&OO0ZUW1KoY%{Y;=o{|_!QjHV+tVcf*0{+6aHy5n6pORQr3tg&@4Pau| -zMd9jFEwHOT>Inlw&Rk7%;~&))xlT#!)KWLK&QhJ({KToRYZxS9SU0Obq4$7zOGs|9p>sUQZYs -z#_?*Je;s@}YxELs+oLB?`6wz|;W;gF`Hy$|k3+#kVZnb4`GWu0TJ=P54SGIur2=ox -z6d77w>;|l<%%#T!2qX0`i;{j@fkkOvpf)=3TtLdTfX)t_k{#!!>+P2xET$w~z1Nyf -zOUk~4>*$Sa_6M}WM`Sv3M+jz@q~=T%87GQ`mg{LwDcD|0o((*tM0s@q;9|kyG*q3k -z>xi0F9ZLPr#_!#yC64cC*W?JnIyX|d_7$%Edd5!DJIjqyQN+;kp@CK@C?Zi^3|ACl -zmx{GR#&5gM>k2&ne -znq28CUFimG-M+0b_D6x^$j70+%MHe4#Pe987A$Tv@ic}^GZly1O4JZH&y#EQ?yhaw;H-1F6@QpFUO|)#)kVP%uWb~4}pjv>br5}*{~`ElQb -z7-3s)g53{J9}(1K&ffF#z(Oh#if)L02StryY(2qN)V6^H=aX1YD0M!hync)jTt#Bp -zuSkMbB$li1$%ZooQ{9sdL2h?X0t9)E1fh;i`SEqngOThm5`-~7K2iHT7|HA;!966F -zsd*lZxrdi6z2lcktmy%tql`3S6feNX}^3_sX -zJ9(bP6a{vf&7HR$n$P+f%`Lz()I8{$Qt8lC^#wD>ywjmk_O%>dZi`w7tIvK5+0}s- -z$8uW)dHVNqwlEO#30oL>I<%ZC2()~=+!lf-w`D803TlT?Vu|pw!#WoCSqO2J4>z&T -zA>cqUlK}z4kAvwY``T@YBS&hPP{L-1FBCZ5SmKs0X+EEp<7U4C6Nx)|7s-Cag*kZ_ -zg8qU>cKyeY_SY6M<>56&{9ri6X5jf($&c!lHV04+Y!b{S-HHIr5>?uGbJ}=|$K5OK -zy)kL;jfK*T=ZCddE7<=$;s%C+1MCB-Kb&qv)j|7PW=bD#Pao?_AGaiZj1l?!8P*gZ -zf{{4qy+$~|x9cU17B-nUTc{a9H~=io0W&T@a5t?6S7 -zwgrKslx*Ly#KzPppxB?6*(Xg;p7drzW2${pmT;Xhyn=c0fyW|p?-IFZkMhiZ4P;T8 -z{;qJnCtSPPv~KY7J4K8oq--A1qzr&VI@ZaT9p#!sfC=D+Hf|ZS3Eq4x>C0_3`0k0_ -zA2Zp3mI=#kyU0`ea`yc|i+h>PLY`_CvhM|2#w@q(z?0jmhq`#vdM_FSe;@H>#4)3# -zoFHNTyO_UopR~dVwzs(Hd#~yHM_E;!5pk@)c$`aE_t0<{{mG#i{mG$+WNpfJDX8<_ -z=zP|PoZbhAwub9xD+o`4>5W-)NXmypVm6V@ysfE|G2AN7)ZKtbO=|7wT_*1EjB$RcIGI{fn}3a%p0&6-g*lJ@hF^2$EpzFYCeL|~!6`&{J9&;ZdCpF95FqU+;Yak*bjzo@Wp6Ru*dSf& -z(xL^+vS%Bo)Hycj=NuSE$>tnLWyU}RGi9q|&MPu1c)bd_xc7iPhe3NZluZl50^`Gv -z3$jHos(v-l`QEaE1;^`Pu)e)VldtNw=F}s>mC$>z|G_@>Y+-n-H@5<_h`rZkgxq=y -zbqzmcSDDzKp7On6=Sx%xT+YyTioMd+^F;Z$u+p|3U-A*nZ-zDkYH@;WVA;2gP1j(R -z(2w%V{!g*tW-%!Hvj2y+H-U?)djH3VVHlRVzzpE1fHjQX>qZ4=SuO=gq@u|p2nfh_ -zM?`VK&*$SOqhcvEwPLAU>oa%krjjHVE6t@3py6IhAInk-v0G+pNbU;rd!Kt}hOzoC -zzyI=j>2S~Uob#OLY|nYlInQ}c`>~JNlIxjX7IO(Eyc4~Ae{?8QKjM4zxBmLE>J)!N -zBIlElzIAcsOT>>*8j2Id?;EgdalJBq7Lr`C?)@4P<8g&}bb#SIb6Y!X>n2IR_gcJN -zJH3@os}GkpZ-ry^acq9$=fhC-7p*pXh}ZgQ?xf82VGj;4tT)$utE@RZN&3B4?&)gr -zgr=`EuQ1@(;^~5^U;f5>^(n(N?5G>!{sD^)MHzPE){u%@v7{yLQKZq_d12n4EvdgWt-uV5N_M5s_ScWlrIoQ;@VJS7rB5P;ZeVz++Ye+8U`@CJ<+kZ4+x&TWpl>h>2@yycY@&; -zj_t3A{n?st~_`NBt=J8@<%vR?J}1xQx13j3F60v3xRARg6<=*YX|V+^b8% -zYGt2cZ~%rYW0j1e)u>PhDbxXqR;{AaTM_G}zy*L-rGmfki+()}Rws92Q|il0DPyhV -zs~$Gr;#plvdM^p_!+ziV=o#`w1#;8lx6qGSG@UJHvI6;zTBeIE^4@DKBhf5 -z?!!I=n{~Mjh>mOKb}g2;`iE{4T)i9rl}=xHi+pz)OMd3b<7^~A{-Ye?uKv@!=ui7g -z)3OpPzqfng8eql}oVb^*n@>^0J)Rgqe=qxV4S8$6?Onlh3h@i<5~J1eS8BfR?y?u| -z(o-(-?RW9s;H9E^6+gUx*w~3glY-}keXin}0l5AutOQL;Wmsv(xx(-zcnYNa&?M>i -z9v%b1OCU9ql+lyW#WCfuuMIF1nsdKa=C-CvzxNW=uH|ZFX)<0)G~l;f?UJij$luw? -z26o|PnDe-ZrpsHePCmyjxs0t%7t!1;6d_M@Ovv13?!hJ4fH|onQy_h60K0quj5FOX -zoC7gB0tm85{LI%$TKaLI9E6^Q^MAQxSOVx*Sp(1 -zA}>UBE;QKowUm?ehuryKNa~0-kLs_AquX3FG>tK@mmANoiou0%bJdW@buJKx<+mdKc*t;AIKe((W;mf4^P$~Jn=WWDR-iHuEes% -ztA@JEJBZHwknVS4O0Q_Ux>WI2UNdNI#dbT-$W&T@x3HKK0E%#q60udVeO^*53} -zV+sgjt=BTW#25Q-BopWaZ&7i5b)hL4&R*F{sK{bB1yS~zJGZ2``iFUFg@`s)#Qqhe -z##d;o7ZlgK;uH1EHxFb2Xv&irqBD&S6mHM67|&xf&ysr{!sn!Mev9!uw#Zs?&&Rqd -zuD&|2(|Dd5ND7&IUS@XnWHHsVQla2K!^(`E>CUt6Vj_6P8SZ2ut(j6+yUmBkVeX0Q -zhukKpf=ugNKULz!l+hI1z0YM*H8r(fB}QJu7Wq3alp5Y!gG`acX^QtQ&re3;2|sdJ -zrTNr7Ze83`Q|u4BI4Zba)2Nw5clC!tPvA&x9E^NrAa>~ -z*5Z@WV0}J;-lUIPO5Dh{aV_&j>i1#Ij|)gCWhtc{Y>Vm5n)T!p_s;lbffrnNmHqsI -z`z`lRUDK_%m2pesjJRI}G6#c_SDd8Onia2}Oxbl4d{Gi#{7?Pm@xYAIF-f7!#K)Pz -zF^qrwMt`zBEnV&6w8eb2j@Wt%+4Eqx_v1pbN)Bd_pl7l8h7{!7i>3JD>vB+4EX+Uz -z@ijS^H&4RSau9P(j-m~#-g_;XxYAht<2+~wjRP+=Cb(Ln+{<<3U99r7OZe>`c)2@4 -zWr^~%M0s@T$oo#6X_xWaHt!?i3+6lIC>F4w6E -z#B2?v%F}+6+P{$B=tF5jDGPjx9;p*I_%v=WvxSb(8zfXjn_I;Gix3*?eW1T1F2Bxa -zsy)d{el3(qd9EZ6v-}G -z3-=09y~_|J^BYg$ZM!Xto3|{c->2}P9lyBw{lzR#(L3N!i2oLjlc#x&pK59IvK-mQ -zuJ+;csnycM;%r_ZJdb;s*Gks3p3$3ktv+mN^Qak7B%O8E6c&lA?`|A{_0kFb>fKY? -zT5wh8Vd$JKMK966dq7gn@SuFnvn%S|RzHeJWdjkZrhNEI)D#FvSRqRKeh!?UQ|iU9 -zEQTRK#o~~%#cREGxu2a}R=#-o#>V> -z8w7XmXrR3C#;8Pn%)U)xcBG4l;GcP2D)<>=CS{Q2|-!RVlHO!%JSvA -z-K!@qUSG3jY);9)yE@79;*Ad|@Z0pCFv|nV$o<-(UwSm`(>Cli#JVWV^;+{b1BsJG -z;#jgUO6!%8Uuntq9-WB35P4pEby|_3?OIoRRpUE~rfXqcd$cgjYs|zcp(83g9Z?O& -zYY@Js0eD?rMbFn^HKn8ruo^X3t`U$p7vJ38dF)}ZOYY(i7x+P!Rv#e-5SC3p^K|b+ -zI93w0i{XERT71%bF%ATE>$r8^>!__7Iz_Q{ak$UodDZKxw{ZGGU-H)assPj2*{8$5 -zJ*lr(T?nDv6<2h>G^aM_`nyE?EB2}5V+@J@%6eSez#|V>8o|00-Ev*d#wR-7)9q6q -zcjGTuk3*5HSDhC45XHPvYflC(UAk_c#b1%~ogWqVDR+oFtm}MF5!JPbdL(jUQ`fb& -zo1eLHtLD-Bx>s<_BxU7U>smoS_ge6oolhv$Us%umw5aNE)x~3GN`b?=*g1+=RuP-0 -zh|N*N&Q`=`Dq?3TV$&3{GZnGPir6GYY=RKyz8HM_3g?gew9i>n| -zrBF{$sK+VPPbk!I75=C~JxZYtS9H57YFu5qy7H?^V_tP(p7P)d=CL1j12%FLE>^kY -z>UL<=$5DrMpO&qtZcF#r1;?|;o1?sSilqu?#LZu$CePvh!E -zEN(Fk7`Awq0JjhRiZ-0BJh6@&IjZSk)QPs3Z}Em~(_!3E9SuiLh9OZ}V-=hd8~r@i -z=_t=qqt-mUHue)M4rGFR%}x2H@oG2};9483=I*GQK5A;%t6XKhm3wlYxV1ZWn>F-~ -z`W5T=JFOMg@fR!0t>eFoEwhe4qAsmqCD?#hMM(7MTT`Q~G%C2U^A}Go6c#R`9^2{n -zV)KGUC#M%SwKVDPX>boje>vz};i9XBi+=XYx#^cxHg{3=TqI3ieu=CUHz5d?$3YMzY(l?MSmzL -zc=(!rMZtla`xOMo?d+EqJpN$6EQX%N@TttB$;{JY%tK(lh>>g3w -z_1ciG^+URL4(U2Dq$NtxNSr(>%cESo(5h;`2{=W@-j}O_e;0fA#EJ|z$>}#Q(C2il -z{|yMqDMTY`3xH$kf%;d;W^2ghdDJSSv37WZ#U+5=-EW##fzD7e`q)k!D0sE2;RtaP -z(jj8=jk#^XN7@pG;2UR=53~dywYW@bbeSx90@Jr7(B&>uYFy%Ka{O_(n>Z5B;+DFF -z>}_QH9+vz$d0OBvZ591dzq-*_C1PW$T}{7x175Ph<(I@MRRFM!{VH$5kP@!tSfL-D -zMz@6Maa*>bpSs%`-22Sy%GNDb+65CcT&;XzWfQLT;BL^3N0jh`4mp0<%H3Khyt|N? -z|LgekIGw<%cD);SZ6W`BH%Au|-#)x6)Ra$_xpBWOWS4b|nLpyvNuXOz%bk>UKXSh; -zY}{m}DAq+c!7Jy&Le6I{lbe{LJdf>igAaDvvOlVdh&yD7I}8TW>OC=j*eTcA@T_#` -zJ8Hb4R(Z!N-n^ps>R`9RQzLup3Rb`!znbgJxL$Q*iV|Z6YxgU-N18pP%GA#ft`&lO -zutVO`hE2P$#;M_;8YB+%3B%qTK+me$o|W&EhO{oUty6I~bpJxfdtf2W3ysRBWW6rr -zGkia3``&}^3Gz2rH&^(vfZQ|ZuW0FioW~C6v0uf%ck{*FT;oDM)SG|Uo8EtCg4fML -zukh}Iu(4W9?gd!kvT6l84c$SU+Ak7|^QP-6!*u19!p;Vuo{z_FOm -zu;w2An;slsd~=ER#EnmWNI&}Lo0eREY^-El+Bi|11owOVq<-zLpkipaW-$x&J3U~e -z=~tOYWVW4Hrw0DWAGy8C0#$6PsPN%$+yX7;w)lD{pWV$Lxx}qN^C)n`BKQ;5M9#=a?{V#w{MZH9p2i@?JIzxk@!jv+xhFpBQ%^oo6U`83;)$kYPay* -z!sh1|vbCK!I1S=om-h|A|FN*9R*|VZHk>HL%s`qMZk)Gc@#2Nd6N4!gGkGGDmQ{ZE -zj{b4Va`?7>xA#M;l3!Y}2EbDjHA$FpEw#$GM|ic>y8JXq8OtbI%st<1BYynqF8@4K -zS@|zKijLk4w0_!ynjXD%X!-G8{UNWK<30MrK@zRY1p9Wq+fSJ@S0!5LUZc6t@X*On -zEU_#{qQx$6A$}pCQxP6nC;sN$NcWc4jvBG2d{%gO?da}V5uLSc>w+`eK(V4*4dX5qThR>_>~rgJ)jO{3N=u8joG#hr)|li{o}{EK(>zN{!^@Lg -z)AwHX)ELX#mAM`<#U4+0dpuL_5nGDu6jlR%7fHX_Rzr81s;es;&kuK>QO515@m*N% -zyP$NPu5_KD+_wlH-M;hjjgKxYJhRkzx~C?~J?_fp4PRJ}Xj_hH%iCSMJ$!3Cd_c6y -zYN+w>#z(mag>N}CaL_%&C$7E5uUeq@wOg3%UxTawjd^yrC2A1UhJ{xc0`w*Nky4(Q%6IIAv}e -zx=8fo;M~a%8Yd6xp8No^LB`0h$OG|HzQ7$h%BT)SH{;eUfFRtWWda`&jlWW!Ek|y) -z9f7?O<#pnNilz+}WM`sQd3G-PtZ7O>_feGM=4YzlNy|65LuYqU04}DY;7W7V^_nEt -z*6GroXPF|yDLs7dDPKUQe&uNUF*AhLqVOL1gp^y`&L)xMXTA~XZqvu;J3Ly)5@^Q92-(cf< -zxMlV{*;?c&`gU`v3r?mjXiiyhb;bfbF75lvgTFlT%TvEh|7G?s3-?j%8*pihwPA$W -zJjTzr+-$*PLm|&Dpj4&gDtgG%3m|6YvlyN{RE4eT^}JViqf$}!^n%?r5tCKRYr?rF -z7f``%<^|tuP#vCv$JqA9F+mZ`$kEKWa3*dPGb@al7r{LLNcT}ZgRu1Glhon#70RF$ -zrKNu()`B%=>zWN$4j1CO!(lJ=ZR_E{)|=MDBPwrL508z#jx|)7O9%sgRw%=VE!a4$ -z(N$$}b;E$eehypWK3Pe&lb5+$Cc2pqg|tjmo%Y-KP@}8b;_5EP;St9VaYPcwU5cYV -zZ6t9#EUun%91(Ghh(kY*IG$1*kJAw(j+ez%Bgdg%aGxB3I4p^yk>Yp(W|_!Nbcte|#4T-UjC8{4U-~Lo-vX))yUSlq;ddpx= -zs}+~JFZ$fjxcw<}My*9XJ~8(3hIB1}ZRXMwrwr!ITK1pr7ZMj;H<)MFCdQ7}KdiD? -zO82=0UlNC*t6bU*CqFKo&HpU-R@6|cpkq}lno?3df^Zk_5;gGHr-&}uWWl~$+(ikRkhlh9A;x3wX)%`)=bcb$+=BUe`|?7NFM6O -zJOkej%`a{E@XTi|U+89i-}yy9cJZ&Kewyx(A>GgPcHIi;ex|2nTulgLA>QIc%;tcm -zrhuk(hK@R2HyEjVVACC>Umu5OssPNbqeb$Y$Gu -zwQ_K_Em$Q7vk;WV!s-^rZzN-Ul!?HBYb=}?RZ0;cK&e@xXFszs1Z(0_J4y8&t1ZCK -z*0$cTa?i`Lv$VvFnT#zb-nil-p{s!SEoEmSJ8of-a9IlSDI{1_B>W-=lM#g9(z!)~ -z2mn9B!5ehup^R}96X8dc5uqJeE~`kKDOH8=EeWO-i79gMYZ6Q?5|gDMms}*A1ca7@ -zf8uR=k+5kFbZ=LF8ewK?kzkgB;#3lxQY37YgK;u%u|>jK0K_SF-rRGwTyzn6x|ymR!Y%J2@f24K(4>8!$wjo67v2FitTbAES(T>@eS~-Ugl?Wk@Ey9WW -zB9zY&AJINPf(#!*M>slPjut#2tbHOW-fA1Ahyr?;_NT_--2VCGq7f|ehH8njsXFi_t8t33&_{2_gnvJHd%7X8TMv3HhnWVQEVk3!{Q$cd8jif=# -zy-!F!29kZ{Fgw>r@LhJWR2{(o)c9z9)UkG!->&NlHBXM -zXE^x|Y4q-*(vt5xj**VkQdDY_A&VyxRPgoap)2OQhK<%@U&}DD0-)$Q@Mk60Q_gE7 -zP9jt3o(pd)rLSni;)h`R=%4FTv;B>*(jiSNbXCb(K!yUc#R>=x>Sf57Tpt9hKQKkw -zA%v0%oeR9@6wR0DuKIsW_pLwBT_w?_eji$)Y^4Fo>B1W4t -zpM~r0;Z-$w&=BKZ*%YHq$Y-h7JBl&lmDw>EC}SJJVBhv!#Wbi3;?sIZVzIQkJq3Bw$i@poC3Q~w9f=)fVB(is;l1oh!Y -ziV9~OCG>OPuuDK&eVxW8sQNf##hWlnlg0^5!zjU{4`+UfzTt!`5nP;brR+5)+-l*@ -zJbU?I-_LhD;Yx&SeK@@4(&dE95Ux0Ip&I%(2_?%|YQFHZ12EdR~~Z!SNpaCqr!T<4UBe -zoJVQQ?-5t0o7*R7q>hKQB^5&}s2k;;;V&VnnQRdebwQRp4%~Bpf!j~gMSF%X`EPJV -zf5xG$gN1^5k@hh+l07UH!`4SAA-4|~#2<9V9TR3baKdk___@9O()dVr;DYbc2b|Nb -z7M^wBgbgOP#VK8e5bMB&;^`0PbW??=95~$9{mdCRUU=Mr!(-|EFWELg*N&*2aIaszN8NFHD5VL$T4 -zY!)z2+UDYS_;PM@z`UcbaCm;lDn*4UK@ZnyVg+CBoJrV|Pi=!j*Wk;!V=SCh!<;vLkCaD<5$yE4K<&wcZQuW6jrg&9Jfh8oZHsBm0d4Xe5ErqUBepS -z;N!~HDT;I5g$F-VwmwTvrGUE*r;|xAf{y_p`Lz(=Y3{E(E9J#2GF= -zsKYu=#7!cib+YgKa%T&vagqkg`*IyNxcHtg_n}FK)9?Ahj~CuN4&nYS?b)`wi56dO -zZ{he=)cAII|M%toQ)oLe3DR18F%N;M2m+RjAzkIo)wrFbcU0?2yJsOR` -z0b8^|>^r{P_I$Xc3q|^kzTCEaisa%hZt$&lr!QN%J@9fsCL4aeFRmN!lK7>1#C5*h -zCxyc6dDJ~{w9c2QDPoEXxgE#@8_2k1um+WQvZ3(czK73kFFg3<^S0`GKR70KhgH~C -zcyQ0us~;2|{5NrlTn@J;1wN!IW5+3~Hy0lKbdK;oVkq%sBhtB|n)SFG;~C3aQJuku -zb+I2^CV6c@UODI1ORW<|l8r*L6zaXoHP)+dBcs%^w+atF9*FZ))-WerhQl_D&fpMveeq9Vh=$ -zc;akR$~!kr{lW+yE`o3eI0Xj;4u;jNlo+XqvQBD5s+-u^RvRUMs=qoz+Uvm^6*b(m -zg#wdD*PwEwEjI;rX=T}%aDas}VJKH|>4kXcr-t5T-l@YIp4_8(xb=&V_$>$;EZe`t -zchhF+`vWV!vn6=RSbTQK;CcdHB!Qu~l^j8cJel{QwQ8vrFXdH_%ZCa_^fxJpT#MOV -z_>HDN&E2kmx^wgt^Ss-}v&aAEJp25a=iL&|dz^UQ -zW8)b`?cDwps;cD6#t*C~f6Q$zB8(P?l46#Je!A_HhSjHd<$m6)H&LcXW4h*OBNqi^ -zFEvEwsz{}nbolNn(1w);WUnyb;%~8(fJwr(6a@3&BVPL6 -z_LXLnzRQ2UQFk>9fL=%(s)ttswnH8veRWAVP#pt+KUI?9)W$%`}%u@SzU?_Bv-I3ErKRFPib?%)~;}gGcK=LYa9R2B`y>Q$8xB9 -z{4j81*O&L?3kLyXhuUG>u>xU#4t1_wiIBcTqx8 -z-t2?E+)qeAqXrv5fR7+?HN1ty)!^k^qB6JsY@0Joehb4NBqOlwPT=cw$gj -z&XAmtmkV%wP(3ZDtfHVQ#sDOOq>mhH;a)0`bwtqx+?)I~$np7t@#r?90szF>W@P*( -zFviJDL^2bd!z9jUavwAm2Xe&)!pU58KUcaJ<^$Y9se_=u_mI#$^x#}+!}cBt7NAVf -z$AP^|g82o)=Mt8`LxMTzxe_J*HVJ0fDDk%t0z0X4V(}(|gy$M=QUO#_-cw@OgL@hW -z({M415XMT5!aa95H$^Ixc-x11{vj_cRig^%+dgQOY@9W~qgnxb%O`u8PE9HL=MLqb -zl8Q^;^g%Nr_G<320``WF;GIumALAkl*y}#nl!o%>Mk%@B1;S6+6mC+ASA4ipwoyS} -z@yTIHvsjph>PHHm3$LoFsu`$A3yEBpx6VaBSDi4jfN~GHls(DCFv8FkqnoV5*-gmG -zmyx7AnHXOhO$|7d%6eP -zB)M$S#f{oL=EJov4@rI~u(vyc3i@YR%O*vQMG?QTVv)2fMkc5&hU}HKRjCH|T!!-~ -z5VnI%yBpoJ2|Z6u&#&-rk!1;SPmaOFFE`sXt3Zk(b(cn=W%2OsPt_s$VG68PQ+ -zt#fyd@Gb&kyHC42<;UHgBdjOEvp&^aE-Bq`vtbt*^^J4QB^RL%5ZGL^O|DS^gptd0 -zgx8P;%Y|=!xQlayl~O=F?ZaJ|Bbkp-gT!xqsKH=`6Xyu!GWu&Du5*s?k`$m3=&;A7 -zPx*wDOL-g3ky}XKmRxhmFEE0}C-)8BX(`V&(?9#5zT>*MudpCwe?owr(#4$slKznd -zCURoXYo)af#q^`+3iY& -z3VY@_*7$MsQFLTkZwBTgCye-okGus&#^!3ASul(-oQc4j^^mF%-dx=r{+JX+cp80> -zdw&jpR0=wK!~|PR0IWr9|}wE1abLWTlcTMJvU^@{{^IBpCHd -z>p3g+R%M5k;$wfXQXj)>IrVw#SxFv^AG$4{88UfLe62-Ec}HbB8$ -zn9T%?Z#!^TUqp2txTDXax)0pePp8Tc+|sY0Y7X4gAEp`)+|X;hTMk^;-=o?NT!UBm -z;O(6Ux^6zHp5_G>sh2kgL~~EgiLBMtt?O8)L%)bTF{5RB(-zP8hS4}eZK~5YS7o25 -zGvEV9x7jO6*Ll@bI6fO8d~WrW!dN}Fn()=#}0ce5kc>#{%4R;@HNLGNE-=#qzA(dyV{E2UDOu~IYeHV>6wc^Zp6yyiju -zO%1>BDe7%jDhA$fC{yKDD;1!IKdB+DUs_>U`-`sREJn`;ZR|yB_x67`{Gw~w{zKDx -z-TXV|545pp+R?bu3vGK55cSs5^*YOkR$y>oDfjV>N|Rq8XEjNaB4&UMvQ)ogBC{yF -z-Z2(dcbohMayLxm2(sN(B*x+pFTYD+nuN%hz?3JA1`>(8WP%ks88A}lXOp;I?(gt{ -z^y~G=DHD&CQ|iR`U|Uz|s>pe5c0z1+$t~=1DyxRorqsA|ADis@PK^)uofHt?_8~AB -zM0<0mOnjA(el&SX+#O0=)JfA;(tx=$l)GHLOOzyYnC!N?ZU1iB}Q4!Hn#1DRh3D2b~fbYt3KQwlMshb%A0-_ -zQBY|Fztbed68v%>@GDG|@yiJbLIY}FgGqRjM0iDpHM=HE%mFXd;~lj+3w>8HFj7&> -z!F@A6a&@1S+=fMK6?9{hb`dTqC43ScG7GVz*}GExMunPk-XT3L;aqkuLk?aN=~I_1 -zB*u_e;=@KOl#$slk1jO`VPW{EG8j}{qjX_LLu8n$L;4_mdq~kMV}G}rjPmz#tOO=tD;B+~t$H)6>P-W@g*L6z;kppu -zqpwH{3+el?I1ACP~^L@t=-3?ztN7jcgJV12hM7zJ^RLIB>+n<2+K6xbKSWYv+)ki5~O$MO7JNbBdt64JU;1Sq;uz$amr|4DHA#71X~{>CMv|Q|52umfgUr7}O-3~?Zu`k=Jtg-oE5v6}XX)vr -zCBJ5cyeurZo(1QWcPp%Nq&9$HI>8EQS&>Fv*qOXK3zpXOR3GjbO9jaR=}t9tG)g$c -zI<$fy(CnN0x&5p#DGLwxoXKM+`EX50%cqkH0$-9Mu#W}JzWCw0*{Izt&TrfdsTJ*R -z9h-1`2L&z0C*mvZGqsnc@^Cv?VQCh1iJbBnPFIN0HhRK`EOBK6PQuPsQ1hk3zo{BA -z(%v3Xo7tSr?0k>ln;oNx08Di)rAa*vcU_cBnk0=#!{M%2&CYrAY+pazz!FKyj+e;T -zWYrp!1FG#fXIQNS6Q8ib&M8VXWp9urBTjR3hm|gVjjfVo6(G3z?b; -zutpoUXf!ql(Ot1^f%glDq*zMMeevDAJF65k?ozUQNX~r$sh=HbhrrDdC>4|+j*3q3 -zI30z+;qb^Rv8~|cngn%0xReIOHo1GgLHaaRhPa76th<~gCfhLH& -zD+ZhCdO;bYw+$;ZG>vX(*J5`c!Ipxu2Kj`$piF9m$fmzlr?Zp?m(F&YttCCn6UR`h -z7d)Ry8lLN}s54<>2dTQ=CLd_=txTk -zHl+-TZ{Y?7e?EnKL(pq?6}D$$WfayI{{f%ohyTm?pi1czhO+K?KEexRrKd_K7R#n7_zoO*g&75dAfELU|cXuQSt~G=+oSvwR1AJj}yC$tI8)^D7@aeAFYy(9N2!& -zuuAFZfevuMa`usRK_XM?4%>haVH=>r9h|HqP67g5b!}CFtCUCRccStuRpZQmYhIjk?_*RpC{S -zI-xbcj^ACkdOa=&xwGyPd6HU6qx)nNj-Xmy6xI9kt7^?_J@G_=7xq4c-T4NKu5N4H -zds0f=tQTD!DW$f_*^wW%=w-@vOrl3eEa^tYGZfh1B3?X*Wx5n6Eb`@?t@)9Oy29j+ -zWD+MjSz*U5elMR=&q3m62qI#JqWBvvxq~Ss25$))^CP2m=`q;&mGLAY;5l#SQ*Mjz -zWz0m<$E19dq1y*odWLsJDl0z1wD;e)KA*v@fwy;_G~@|=SpLT -z5qI4APH&)t#CiS3Yue#!9>=KX2xO#K*bgVB0Ide7!+XOJ=-9|vs%t{2dP06=ISSP= -z3{=?j-t|gJ>FD%ED%=r4D!SAA=#J)y{PH6`ly}X69=Q7Osrs#=^ar^5&~<&N!t5$cI^FuB(E4fYH+-ueiFyc<^p#?KN9|!Tv)>d4(I%tE-H}Gp^L*(dGp189pd~ -ztkI|h*l4!-qm6nYBayosze79IH5^66O=_jsZf*9@7XtDrug!_Nj*U{UOH*{Ll0H%u -zjc2qiXSG3frYS#^+%azoi{GuC>4k?M#Wp0Szq2;0@&y$VQ{E)`Jt@^}MaQq?qb(d71$cQ3g~V_;ZDoRN3^5>dSe8D4}Ov -zkR##cW+Y4_Q;D3!F5B6|#M75jJZ9CN(lxi|3GI25*SV)M9Hr~Dm#*!;(h>ITH0aMI -z8vG$^^RYbPI0&44ID_)poH$6Xpf)EGV4yQqf#3VceI>~Y+&%xqy7?nGk|kFkPwve- -zHzLUL%H8r|<0UcH6_{006XE)}@#S@a|Y^!F#6L -zK9vlHts$t-H&CBavQn|KTFJE8-AC{;7vC^c1ap72l9w47uxAbb4rn;y09m{*^w6*F*#xA54`*IK`C9f6IR^hoG#4?!*Bj$x+EtoYj#dI -z)d@qt_s4XxPFQiHbGoOTFud6*UC!9Ngt2*x$L4Y4@|aCDb$Yq~qMH5P`fuS9ZnJ-) -zX~#5q)rJ02NQ~0>H6UvP*%TM{ZEqQ{jA$6_Tegs_Z@g&>6-wGiZC_%GE?uHmLqKTl -z$VOe=2S|?^t_BFr3`G^WE+et%Du86!2L`p&FKhhA5dJ1HI|h>8hL+Gcm5{BGN(iXZ -z2})Pnq1Ie`_lH*hh8=p#3Ch0iR7cFFp`>1-NVbV2+Yr-^V5x`x=9ENy1w4|DM{9_T -zBpvvIGrGc7P~p52lrNXdN(nzWLFqE7xMVCan+B7-$B?{ZNZvuF9eQm0I~CwfDPPtD -zxe7;}5HzW6;gAzVBsD7>aE3?~OU68E8*9qFxtN<|b@ZQ7%WC|zKO<~c!mlO3Aj1Z7#L{)jot4Nor@c{S`dU<$&Km^4_*l5(Nbtt2{K -zy&Ij0#(@*~4e3}PXLys{SqNej`C1Kv@=G`v}KHR!KyK%JwwLl -z8U{NV<^f*L-7VdrUG6#>Q+2wvc_2wIl%yXyA}?f4hUGS9D83`(81ok%8X&(JI1R>M -zJU%cqFHl&VUN(&M7@Jlk=0Tqk=B11I5?h343_W -zdn7UAf^dj@M9=b2YZSuyta_zwmYrxl_wSs@7Tv{59ajiLC(q<0W+}cqCRcehs#ji1 -zlzp9(ke)uqLz=E{=E0RIUeOk|Nz`XuFubYb8pwA|xP#z0`8-Hi~2Qh;KnNj^oN&T2wUs6&pCdr-2Q!z_W#%$GarY}X&MDL4*M{{^CzK73Y+_i8&A5fT#wrZGyVo<@F1olkmS*i -zsr4gycrz27v(X2-<0^%IH(qPr=Su&_`ixi3+4Il^##rlB{NvW~_fF0uFiL!M8*AY_nGd>iiE%gVfglJ0AD$z7hQdQQGJuRz*PM -zKHU3OOyPE_=t~o2)rk)!+&*L--*fIM!em?4h2|KE+YE3EerG%Q<&84GwDsx}q+eij -zF>j7^j3A*f -zjn!fZ!6Z>7Fn!?sG|>lO0F8aX*=fAD1f=c>gZjW}X*5Mfp+jE(DL?JbV*4$iIStYr|ej`;_osI58 -zR$~>q>Z{GFh8N%rIb~ -zXbIb_*0OhOU}?XsafZR(*4QDoWRpEm`jUfB66D^=X1l09UH!N!xDdPMD+1u+tH&y{ -zdygzyuk~WJa=kbBHaDB_QIme5EMMj~F*Q`YO_sZ)5A%2AL|AV?yk12jw!1fEZ)!qX -zc1{9GXiOn282dg9EIZ{Fi}jr -zu$J6y_$Z7%-%GRt0!zjta`;=Gq@Z84t>MW$z_#}`Gue?fI^osSBZuXg1rs+HpN)02 -z{W!He%^^=vW6#_9;-8cFqQC7W%VhR!uMm)p4GZed?p+-?btUbusGg8bIKU>x3Ia(? -zRnY&i*8_C-G1<5&Qn#tWwg9B=(x<3ZQYE0ykGrAGRK8ZO(ouIyPLWz4MlxYV+v^gQz-EtaeA<`I;+G_)wD7Tl8;b^b3a(iJa -zuO;=~GMhBIQ;YE*Q?b4ZlZ4#+N2Iue!g(aMXQzbz_)lm-ZnF(?n}K}26ve4eiQAp} -zlq<2IqbH%>G3QOUQgIVx6`P+bH|f6|1;iU7?^C4a=r|kyl(5DL%hoxyC-m)5;ma(0 -zYf*9ZCWkx{Ke1*?^6_KH4!)iUoDYe>nK&CGf(V>Rv+LdP8J!v?Hro1*wCj@Xs6vNc -zkMTFrG5(_D@yE&I@4a4O^lXg3;QPkk{MmmTe@{C40VS~a*A+V1JGMfh|D!Im{)FGq -zhc_h!BFfDBTLgPJYo#bAIBuha_&;Iama$6+cFG^I^ji|vN#8`DAu%KW7?Z7$=}6eI -zf5h@tG9C09l6usi@UQNDfApWcw--Ua|+Ko)24*CvYfU>g@2{+c}{p#*xX)fAUDchUyqZIh*aq!Dd{6h -zc1|x>nI+81qC(kD|2~~UWJ&)u^@hBb5;Za_VK7N&lyoiQj4(-Yq1j#;W>d7nY_c6v -zFc)d*yxx!(Q=)>i5(1plJ}jkWbM8+|h6JfA@RQXmQ=bhQA=^`;G+7BMX98h-=KX2= -z^5xUzwCpUB)|?V`FEinKrW4~2i)r_#wdX5n9pg-VR??m5MEps(mS2&wO)>xN_<`j{o=r|*NWI8WLr2~rS-sX=pg{w#*pPj6h;-xA& -z?vf8IWdFQ=m1M;@d!GE0kN}y&C6ec)|5cv+xZdWqnZghMH+e#SP9Xj7Nj#bxvMMvG -zg0y1d45BwDCXSFh$OKYBfXUr2QBnmZx%!Ty-suRb-q}YPhbA6M~2?G*MViv%S1YcZ@h -zAu%%~HIsUV(r^3<<0HIR>Mq67pej!sfzssKzc+qVuc9B3Ss{hamC$n3Vp3G_B;+~D -z$q(+O`scir8UAGEwjU)8o9T^g)4`AGWqCh{np~5mC~TK^tFToClEsqSO?+?roCsce -za4%%?*tH#!Rb+9RqR^&?7fa_%)b#E&lBzq^q2E)gvSpd?C}F<$gc+6>SW?)mT!JWU -zUqtLT1jQ&;a=3|N8e%)kMLjOT2)`$7tC>dOSVx%as6$~`ianxQX!}zzptpP~KG9wcL4ANu!@dEsjerxt0QlQiE#Xturks -z{7yzrcLp3R!uO#Gj_QUgI`J;z0}piLrNl1{<#-)xQb-dVQy>)+=KTihXWm&!wJY -zcUxP76>xy3pQg|GIYUsVGh^p7Q*ub^FT6A-NgE&#dY_R$G(Y>xtyU=pK?5{aRW~*CdUocazCcag7oPPp>46Xsx#; -z5^wb&!fh!(2@Mj(Nnrp>^+U!O9=lEotK_WqrsvS973OfA}HGfgXKb5EC=FdQCS) -z=!8A=$w;8Q!pNB?^P3qbH_V`X_%l7WrQ^%J@wWYKY)bs62T@jMoUECFMXL`w>9;)? -zx0GMWt!5uTm3F%DvrHpe+W>7zfZ6pPT`FgG@+3?KU -z9b`%DMr_zY{);~*D$Y2!YzAfJzqBQ)@i!S-5kuW?oT8h -zB#CxAv6q^0j+;UKCLY2j7%373r*p?E6vsF?t9s_`Z=JZL5BxFhn2d8ZNK1bX3kj0e -z2Wfw+P+Z|U?oW&CT+7C07>wp7>7c@*BiGHuqZ%YKw%BCfC?fmf2ZtF9*Z{+A6_p`^ -zh|2XuR0_r!)LHS6M$u1_mC!0UypL<5&l6X~e~cSCgStnz{!el1B)TDp%kKZ5;>tYC -zoIzdUn=}yC)Icyoc?2b!U#);ivc$^S`&p66PeBrP-+!9~v&p;G30|#D(m6lYyQ_tM -zv+Q%Fkefn(q_OFH!1(lGGE&5yvg-#T?{P*_%lQA7o6*~j+=w0#tZquyN&Dl{)XeYh -zsaNrPdYYFKBkEk7mz0o?#97hO{`i_yyO81=dL(sdN6)#%vz*UOj?BvKJ2$ayj?27| -zvkY>=Fo40f2UK}2Q}KsGhHdL_L=UC@;lz$W*lFx(PO`I;>delPK6W0G1E -zE#a*sNN`Q$?g~Sk?(7aS)8G{9t)7!&I_W@}7n*+|AJG|bBrdo)8jfDwbM9Qaa|e2I -zR$^ZV!oeHqc=DdWC(){qni(jf{}W8ovrJwdv*e%UxGJ*6qh>$uu6SDZ-_T$5YTSQa -zuc7ow-XY6pG*`}`1|9VCxC|w$XH!B2hT~OD+nC- -zfi4NbeehXHd>Mh`I?yGde;+(Ki7q8@+y=TN`1Qf3C-E;5csqf6_rWJ8(Jv42Ng7JTpyP3`;&0z9! -zm?aCj>*-|3{j&yDGu(qz%~20#FmuUYxRR9J0yEXRZ8*yEvDm7)SbO -zcDb%@i&#LyN2ySRYhZLK&OsR8jkv*B_rZ9LXhN8b9mYSDYegC2n7mYGiHSQPm0=+% -z!#a0ThH)M(X!f%n8<&Lmpg#!v2;*dLU)Y|=&p~CteR7LV_&AA_jyFjFa!>*0>*{${ -z3QMd@jPv;tK$rp6CC2$YIR#jk80T~4Fj$uu=X0d6gK<7v4!7t~VaRedJ6i(CWqFi) -z+o?U?{IA+$rBi#Pkcy&&m5%mUmPjW9c3*oWkw{MMkw_vqw?_g1!h=rj5l>P$wa0W4 -zc507j5q55mX$affqX1=?#^lXnmK1WPbbX#X8F<5_f%gmg)eZdyPgn`~50I?bR6>@c -zMdA|K$r?P)0KzT0@fuXmRCba^+9B#=o_UI2tN4ORC6HI9$c;F_M4H+2+whw_mccxRC1 -z3F&7MDAyDwZzi*Z<-*eSu2Q*nxf6jhS1OlADp!9}E@8N%RQ(d^D0`#Jg`y*+u%nyN -zPf1~?5PKBhg*uD`%Aa+ri7IQ`3> -z(2t+($@J1+^hj~_#5JY6R_?1b(ob=Hmez15(u92p#5uV*ggoNZ=ofq!X1o-4G))|= -z;f|z9#`nPD($XOWtdKWuGy<(@Fnt@X;SS(0u5B=yZOM_3nz`mQ;r#@JX^rq+!b616 -zzBG1-MzK=2viwz5T+Em^&rn|M@7CCGMaa!WNfD3jnc!6e_qBm|?+wPC-By{o#SLk= -zfVS)(9ZQLSgdk15yHG1J$_C;(QSe=EYZ^M?`^4OCphmj&%e|jQdCQOin#Rpwm9wO^ -zyGamHuL(DU)v=FZg^P`wVH$GR6Q&!1n!S&L@x>d`64s?jxuvHNlO%WHZzzX2KqC|< -z6rmjTZu9`D7?pA{+QBS5tw{|d_xC6z=?}&k?-|0v@ewiOeR+JBp+msX=@t3dFd|iv -zQ=S$U8GgStJKBoci+7L%H&!tIdSLC`jOs}5J0yP=Ads>Nxm;pO}s|x)IuKVFhr;W -zhB`FAXQsVmOr{+p3`>?VtcQ$AwPPG3B&h#+0G39f2%(OGL`F3_${8lDdi^w+c^;~#@p)ehK$iWFhANb>@^vqb70zS -z7_nQ%XdIYt?9{Hx7_|d)!iM30lQ9Yh=7^ozWf^lf)z-GcejA3pBx9~SF#GJ(E=m~q -zLD;97=Jxg}JjC>=j@R_O;5DaCSvf}Wb)}LS6+&z@f{9&7e_|Kn$L#P84%+FDbCZ3p -z{0^(6nX4()l$4p9m6anDt4ANw>BoDej3V;pBP;i9s!iT}@5OzcDsb_VylF>rex6<4 -zoJA7mmj>|hZ?GcEIDKiGR~>F{*Zd+ -z``(juTU6{pYg|l_ap+sWKjgNQ_e6|V#8_H`=byz7tav8|TbtH<(};o8H|r(YOT-TQ -zflcgijj7m%t=oh)-Ri=B<+a5{_`A$+n^(IEV&{WYh#i6!PkPB>XG< -zRZ*~oU=`Hb+-olaN+oGu6_J{xts;gB>Ecp(#k5zzR>gEtamnw@4WNDB@8|pd{{A4$ -zo#o7#GiPSboH=u5RP>YK(JMlu*9C#?XoM=;GtA-9JT#lF?Dx=&Z5P&qYRq -zZ496l7F`jHG$J?}G|+L0{fOp8bF*R0Y*`Nol+VttD;~7#A&x99_K%5zJ3@xihqnm4 -zQiL?t5`G3uiq0%FTX36r6a(@B7-))cb4=5*Pg?G8<@2Yc_zYkA{L#9aHi`3nc!ap*y<1&^8waNH71A1$Ia*+3y(h7@ihGMkrb%068a{VrT7 -zgDbCna1C*-rtn;u;0v#?$FMeNm2gWmv*2CdoDi-zpwfxtR=6-*l$;K(fAW$wc=QPP -zN+0}684q8&!LN;vJ&+TXrMCjCB252t)`5`dkjc^7Y0>e*L<@{3B?dtDeh -zLo#s2gHzqwp)~06^=LkL=MFxdsmcYA$WUbu3M4)r?W36>yRQ|-HMj1LeOs<&&Qdsn -zoINzAU4!>-!0hY4jvxo>|J4#dRKI0H2LI2&aSihMB+I0sG_a@n#}ICiHp@cZPb;Bc -z8cHKHOA|C`#)=qA3w^F}&x1tsb?M`Ef~HnN(wt1{DWupofct*!^Dc -zeV{9oLO4(GgRohYnSebn0yz204TIx^P%hPKH~894_(Axcfv|LGy!IVP!UnuE06%Fm -z{z$l&tBLzPe%Xf1WsniY8U9GPg(zrTW`q=kY>0etu|PjAT16yF&N-m{yi$fT*~!%RWrWO!9O)bhd5oB26^lb2It;-C(4*e -zw-45JK9@S5n`)A+F`to(u;ri{NBst#&O7~h@G%1_Gyb-h{Kb0*J=-iq)4zIgiT-3R -zcprjP5G8)`lD*ywm$|FnRFiz|3V2gybl-B5%5TLV>ZwaB&J?;v_Z7O~&ofXb$TV=B -zav@0ne|jU8eU)_Pp2lWHMe-D+dk+7M_thF-Kl!>PK;wWP# -zvb3B(H8B%0&a|ZXOi&a;e%OER%|T<2PZ;VSOUx43X}y(8rU=r75AP^FluIZ`2h??1 -zwA2V*5`;A7m0Sz4gv=HI08D0eu4NQ}(6s--%~ERGe`o%YYk4ajlnFD3>R?vnT7rCe -z=ovzu<+&CdBIxNud8Vuzq*d{K9}Tbk)knjW)hLfKs>;WArh~wJK@fvS7@w=6ryvrs -zQ9BDukfagobE(5Vw-a{%7%Tz8O(|c$FxMBCKbx_ejSh(yJe}o5C$2U+ZiheaUywfr -zk09fLTn_GG0Oxq22!5>--$n8g1YUrN*3CT%LAQ|Nf|LO-BamX`=0J*3Nbx|*fO`>0 -zfoMx?E<5pEKn=nyf&>3AURdmy_-@Cc#o!r#z=r@z0Tjjq3Ie^$S7hv76CI+7)=r3y -zkBd&67M-ap$R7q-9)c`I;leO}0sIXx!%>h$K}#qKT2as*jDpY%a4Iwm1zjizZ&O91 -z3ZVZIA~T{x0t&RD@dEPk%OKgvO@*9B6clhsSTK-h3Wh)kd5Z4k{56j*1HP%RGlhAh -zNcWd{0|D~!->;dAz;7}?YN+3VC+}rjkxqmgImlBG*PSCs=ky3=`S0ndqKSq{s)hmA -zVg~l$k1#``=obtFI6|4KbmV%N3>$)-phh^YIryRQ-b7JZZm@aiy@`WwKQOP|$FDJG -zk_vyCA9f$cx2XD;UW*@BR4+D64T9;~Rvc`Ys-cx=Jbj*Hd3G^)_Y>~!3WR}n5=Rhi -zwA#%BxDkhZ--Ea&{DENl-oeo5VVW&k1y7g{q4t-jFoycTipto8fOC~y18qupSHLG_Ft4WY`~SHB;IR6z06RQ$e1%3Q -zz+QGAb}|}ViOVZWO?a{49rR&b4lgeu+*j8L*X1iG)b{@X0>PN_9D<>Z+EKUx -z>dw#xfw-0&r_)IXC$}BGx~4oKv}t}K_+}TKMvBw0dKsVFQOLvF3{Y4LXRZ$x -zhC0x+%FNB-t}A?7f&seE%!x$K6)!88=`iz>cag_FVN3za{9UqD@6ViQ7w#iEhhD;Zh+5nnYFxqrfHK+rBRxm`490u$M?&TF9||TS2ORSJsj!U7sTYB;&YcQ&m|KG00l}9To?|F3GiV(rwo4~{ -zDVJc2PR{z0-K#NI+sq3p*u7qU7`$bXm%*;*f-w9^E*N7JIdlm61+V&mpMMSyUeu&H -zv>IdNC@vCXWI6n3%p%Q!K-~WI{CJGnvxJL+`I;Mt;p;dOW4zfu?D?|_=1#WYGz?!! -z!bd+O(3chUQeK8V9mC5p<`;gnv#GT6)bOe0c199JFTb3jnpdiNtDBf^H)$GsQAg2K{$*9C<0iW -z74ydx%&~0CLa63!2nioY2U4{3R|=*v+meJ*;^{9H%%OplMD7a(^J%taHcH9l9SX)W -zkdjY26ld2kA7@)0g*4KBAIG|EOSFI^e@wt}pMWEI>^_dH4@bUv2*(cxapaCFnBCbH -zLco!4QZQA6IFj(OJ=+o?;7A@(Fxv)kq#G4XdA21?z>zzwVBQ_Xkw1iREXlS6KpOwq -z5RTBd8zB;e2kzg4I6$1yNAFJ`%=c!teSP+V4`2SkU>(9dKxYHYzsv)9 -zK7Onlf(JN1m5t^GvUVs=n4+27Y`Xq0b?AZSXnm|6tOLp}%C^|j;B=B3jAIsLqvI8> -zR#6M*xjvp^<_U3h&4Y0|z!-7azJW-@n=`X5PcH^CbNAz(0Vv@z@Fr4t1CLlK)vc&- -zKC;*^MU3Gp!ymfH-wnyOPF3Ew73aTk1z@FLMu#OukBbLSj)iW>!%WB)-iqC&Kt?VR -z*kxwb)wr&lpj;X3av4pw1#Z+jxhloR8{ncDMie>`sZ_{+_cEgZyHzk@?oe#(^VxTZ -znc>;OAr`k?5bgnAcp*JF8;n>+=@wLAa-GIt@q3y+gLN9st&0@q2MDtP9;)?Vkm|DbxL{qETe7ZU8*v)|y=-FnY7XH9}WoG9Q`aI+eQb1oD1>EuTE;%=n3xn8nPeSvYRy%v>QTYzyxoQy(?<17quqYX|hf -zD~}Dlabs9vW8qcJo(jQht*{10hv3m#1^6b7HcARvIA4W3bXkat`L#kZh{po}GU%DaWxBbug -zpr*%faj1pehITr|XN`Xj)~u -zGm(*!-~eTw!hS^GI^f0aFz#WnzS{_;Te$mUZ`UtqkJO9f@Azi;<@{ -zf%eeaZ!!ma5r2c5r+6v`B?&SKq6K@O7BlHNL$rXDT@R#?i4X87!V|X~yASp@KSL-8 -zZGeQs4YvziTfA@f!``N;h> -z7zy6Y>*@T|2iiDgYbMY(VNTkFP~R?MB3y5Q_A#3>!Owo;5S^e7W2WH0F6P0ww=%8f -zdw3*0aj;&2ZT`u*C;GR|J2@yFDHq -zwDrPi4t>+R{R@!3!AJoP(QkR>F!w0Qn~)9c3bv0EZ~sFC&P<IMf1y81+4(*>w|m-g(Hjy4{l=NB%#^^ej5%chck0BEpsx>tno-4*%Uy`%GAV^ -zf%Wm13kUjIARjX=lZ&|DPr}5=Je*07x*s% -zz^m$ayqXx>#87-x73i`Nj?P44|H2dz2KNZeS{s~GbX*6n(`3@$UvF0OZQ!5;S3MoU -zpZj??D3&bhOmJs4S!s#L1QoS2pnOzjopz@dol{swWRj;)QaB{htpVl3GvPrBAH6J# -zflrx`Og`{Fp3wgRnU+*YApHmX(2|7mDKcsNFZraImf4WN$%gVxNBOW!Uh$WFxAm5( -z2%cmp-``Na8+zLBFZn!r%R~eZ8_Gwbd{_0n^nO0evIX#->}5SCc_2=)0KdTSdRqKI -zT*Lx!Rowk8kSPR& -zkM#je8JM+nN_gHM$tg%?LOlCtVyN%rdW*hOVrI?c&%WeAoj0=qHpu)7?5+7iH+BH-fV1Je<9R2`%{!1^o@cxgl7=nY#kwld8j5s`AYCULM -z{SW;GZJ{Scjulp;L*EF&azE|>4QXi4;4x@zajr7mH|s$*KeY86lF%BE(=GCxKe#t6 -z{C=|B*B&3-WPlqYE&OHm;{sUue}l~_{ACd83>N|#06>2brgH%*_l-X|r5=FlTk8aT -zeQ;MN3sB**x(``3whOKq#2t8b5atl3Yqwso%+Lc!K}7#(0r|B*yzKG6!d(pbvhdhv -z(dxhbl@5cTzk{?P=xmv8?|OD8b|U87rOV&L?$X@;${;yf{8O$ERWL;70ldTi|KJ@s -zg!jJ|aE|}R`_IHtg~glJhrub96lLs`9I2aCLfU-&56lI1qaGerxOoTU!Y#Km26EwV -z@XbTH9-wV1&>&H|+LK{Di2ZMQ;Z;Z&qQL{aAZ)LNZX*xhM{}s{835NM#eXe)cJ+mf -z3yFoEjfflYUlE&)#y${w?z_Tg2O-a35pk6mb|k7h_WMJVzL|B~+@3v1UaSA{z9I4= -zK1OYv#GJ^mRKeUu|J&cPV}WLd2BtAb2E2E2&|mXS9CIwgQjStTexTW?Y;=e`dT59o -z%0NSeJTNds24N3lZZASZx>HQo`HI8HgtE+m3`<)oeM}hWsI5xcq1e6-BZ4zDF|N^l -z!ej~Ye4Bp}n?bM{utV_82@D))fXJN`P)jrhGnW=es*L4gqh3xY2Dt1aYpF)PyIfgf -zsZqK0fh_@t)n`ZZaRIeVW4WU|#A=`5(1+Kk4y(C2ut4#_0o6yqK;XC>!#mH!1Vl>A -zQ*D?;r_orD4=XV~mG`?o87yCP|HSxbE*jVd0E&|Wx}p11pZ)ML}z1oqOTlTj4hY*pq*TFAQJ;7IC4MaTU5)qjo+A8o?2_`4AhyFqd7SI(@iv6mMnDjS9=YXw4x}(8!WhJ?;kV#t)jc4>%#BVj9-DFs -ztq*_ZE2d1!c;E?A#2fD{v81HJt5Bu>rFC$GweS{w*Jr9F_VLd0C6+lTeT$GzC*J}B -zB8EXEhw|vTpfSjUyOq%=vVB-9S;4k2#ZW@ua{#r`P>UVpwKKxH>IAn%tD?jWewt+DeXy7wH`=)RlXm#|N!um0_4&xYj -zKM_()sFZJhMSGB%baLebZ9+mfvp1c8@xeGmqdn=|iU;Bnk`^FXVdym27>3S#~ -z+#e8#&jxl8@SWkygZzIO!{kC9*zPP!#yKIrc925DefG9NsJzn#uKH&CIk -zPu8Vd=0gJc>|j19`+BXoeVI|>m2lH8?5q?jmb5GpI&oUO8z~rRUkKd0IbS)X_8?uj^AK6$=?$7PWG7*wK~_a{Gtx#~9*W7Q*XwjBCXq%~7=!pGrCYKQzDtJi$be5~ -zZaS&IUneV;8pSePVM@eIi6T>?1cX{M*i+m{jb%X&HYS3Un_x_oaRqQw66i`k-nCds -zPx&Q$i;CXU1*XCU*%8dF`DlAeF7!tStIsGIc`CiYf3`BkA1g?XU>=??oH5P!N5`6w -zdM|}eh1BEdDkUx{$i|t8^U>~KUZC`K}U+0I~t -zVHpIBiRerVs!f3>ft2VaArwXr$3WZYgO{cy6jcY+-*(74A;P -z`xhkPxCFr!x1UK!QGnYfr53&>gtsKcF#2*N*v -zrhp$)1!6&1!kiPr^knE^T(a?ZVQ+U4<3|U9VWW -zg8%+^Vf<0(eK8f-aI-Ec+iIjG7@J>M{?L5jdDAGZNsk#t7%LJv{Yz)ScDYZ4i;<}t -zc>xULfC}H$ne+B-fMBLwdo5AlOw^0RaiIV)Fs}wxk&MKkr^ay3t{8>~+#eH{6 -zDAOiNvrOrpN32-_y&d7Qzpd_EfPbKxu29#dRTXUc6W{4@>a3@93lwQp6Md0K{=_2= -zHJ_~)HOu(4pp{CW&&xKE%dTDPEA27Yp)usbb -zH&yDYw5Gnkyj$RvykPUSGoQDE6dBE-iZ#x8}6Q>J2oNxfX%HnWp_8)wUP -z@M$eic&hD)5L~5i!HX{N8@v7JgUZX3DxQ+rq~v -zJ&dle-xmHie1Q(c?}&KWYHm3cv66Un34X!y=#t|ijCf=TJQ4rM5`ast -zheeql`O`Cik2JECHug5lbW^h&_P2LA4O62)VkrU)lC7t1lPmGXO0)SD+Uu?tTODAm -z=h%tG);)GWs@5Lb>6z8qEN+wXhoOg&I-MB1B(l^pW=YB=FEM5begJ|SfASKemqZp? -zMlU(;yAvdmB~m7nwh*s-YQ4GGwWCJ7F&T4+=R2qhr;~f5i>0=a{eT}xI)!&#)HW?u -zPEsnCdLJ>Kcs0I+!k5HTWtkmfe7=#|p81uICEmxxIj -z|IRJQy!3;s1L-bM1qSldE*pt$CZb}KDad(_xsl8j^a@pPR4)fZp3fU -z8gQk*3u(0;y@>slP*X83LltkIg{Qo!#Ko9YXPGiDp=EJ{%>oB24k}3h<>H10i_Mnj -zphB!vFi@qTA^crWjo(g -z98sxkD7Lj$mA#*2*pOs+C#!5*R+Vxyz~1-C#rssFC%|f>)&gRHQttLY(Rh~D%kkJY -zk(MgrhhM(*^OCr4%huTM;KNko+#O=~*LSp(AxLYk39Jr_gjRy=EivxKOtEiUPu(di -zvy>GZW1U;-&g|~34P0~Lj@gn2Mfz@Mjo;(c5(Z!*EvONPRfi-6xy`T_U(<1K)fxmW -zgjI|MOz%^iz0^+v2%uEpSNY7J487hSdr-WPC5@DW%y!#XZF#LKnfbj%r5 -z9X1S_VXVMnry7d#wp=|^;lTp{`>rmF`s$7S!>dE!`3fDjiP`kfB6VlqbwZ^Dv)3yk -zPlp%_&~@Z2t_(zN+m%%IK~_~@+SMC-t~eH!XcxYW$JWcaPnB(AL^K{mYJqkFi*X^B -zNB7;>-`F=R_|p1M2Eo?>@HUCM^EhY?A58!Y@0DWMNW{Z{npTjNss+ZC8i{GZN(OZN -zISI@}VDQj)d)q{nxapjK*>0i=-mw_JyRYlh;7A-(>6%3uXrw*NT~fi5GRb!Od)S9q -zU6QV^f*sVOYfc?4EwrkpvvwU_H_Q=Th%dI;iPEI#P;o&L%>SP8dZT%Ttv(5C__bLE -zt+7oM^W*tn+RbFnxEDd^V=Pl!U8XPBnumC--JFL9x~b_X=|iUMW|D_`Oy1zN5|2Vx9f*Y!Y!x&* -z#@Twxv0D_|bfQ|i5(h}O_Po56Sp`41ccnwB#M+e|$1CA$wE8s8cej -z9r;uxITd#hAGrt)DL35Z;u4EnFBkL`<8Y+fKznpgxzeK#CC$Kl&?>Eb7H{ -zJdTtcxYAf2V$_Fo58*C*Tw_PA9dsTwDxV^Ri^6?U!IbgyGwr$=4?83v5gv`K_k(8~ -z#N@EcFw)R4_2a@WqhSgYF*zBBOlqx_no=usmFK&1#3eZ=;+?1nti2AWp1kIDNw76k -zS@>eO*OY00JHJDee>^DjD}9ZOjK*1NnU$hl9b#8onbkgW2$jiwX*p_bt=vj2ca(qa -z$Q461B?TwqPa;SM|fl@Z3J;*zyD;<;w& -z8lue6A$B;;^?qP-1&9C+M7<0u4_w&s# -z%*(JyovTCSY74?6!83}=NlphM`}+E#6ufRLSBgQJm@J{g@LD3Ds%Q+>f0=437eAS0 -zlwt5FSM|sV`Gg7gYmt{&&I-yZ(jWK8F!kE+cVaE!B^^ET-hh*%OA2m`{~@2KV6u`e -z?qn_mci1&9`#6D1aF&;W44`3;Q#wj8_0F>0oJQsUW0R^YIcL+{aR3uui+| -z)2#N4mH>Bov|Ar@b~NHv@~)~nLcDP%#u61Sdq%U+OZ1n+SgI5XIb@iX8sRQ~$W3j9 -zBGvF7qF=l{4;r0le>?F@szwfFfZEWrq^hq?X{Da1l{Aap%%EOtr8b#>$0VkzKvR~uWa_5z7Ylz!`!i{?7(my{ -z-EC#AwqisoPKLQ^imkNdrOBF*&pmCeVM`Ql?Yb8^G -zVlWg-JRM@shvA-FF@P#5=pNsU9MvG14$XJ4Dd~9PdS>|;-q4G_OYEmQV2^wtyGC?4 -zI5kGRbibc2fl%e7-0`--Ju~pP{)p0dhwk0j?R%e^T(E2@g%7FqSEMY#0v>dfupv+|6LyxpT~63)cP4w2%5$*T4a -zLY=emw)Yv;Sqi~ICRZb1>TUDm7AnDgs~?Kw(v`(UB?WL@N}arE+^48Lj2OMCTXZR< -zSt8t{GE(#Clf8?TCSqPbm3lz&rYPU8$*65`ye~d1E6HhSnBs8uIdU=UQY^M0cJ03R -z`FtYPhrEs#37C`WUcq}X4hqX}6evxX5(HY%6G9lY3^_t;LATSWDBXb4?aDO%C1oN7 -zQ~&(%L2F+H`@c|_RKbfEH8=CCDJCKX8i`G19#i~_-GQ-G -zGX0T?)xQP{nsqzYrfh9uiDW}ll$A=t&NJ3i*R7P!P#*>PxSgs+V?5^#eb@7De4Gin -ztb~rPQeB(@!oy3ui>4*H&KvekSkc~|S9r@r&5@773PypzG1t6T!%}nGBrK6&B@)RG -z6lpnWT0{jr6fjjMr>e!^yzDp$4^7|!RpN6p2a)6^lHE%&AXskIZC^PDky7aG`z@}| -z8RVc+$j=lT;gZWpBQ@vD$2dMr?S-B^^Uw_teeq#u9`aLp`#OWqe0-ztPFCOTq`tB| -z6E%DLhk#8oyc1|8<_Ig#YOIM-H$hu@=9-C^<059Gn#*5mcT8 -ziG+9}5fPos$xkBRvY6v9o% -zCO9j06La91SzHYpU`HE5#MSn4z3Orkw^fBGTmBliaR4Vbk>n|V9dNqrb)UZOrjk-V -z^FB(5-W0(t&Xklk32ezFCMs-;(MW|nc0qg`UGs$rd-+H{p_g4Aj-Jb)Q+q>}2RrOA -zQQ4kRS(k?fMPlypA+Y_2n~Cn_BNoWgat^(J^XNa|5#aHmxYX17r)T+a1P1soNog7% -z4z9MUjO_T~#_=OCr4FR@D%pw`(sEKO&*4%uo5a*9*Qhmq*ny>>?2;N=|J=-=^foE|sM1_vtYN@8n$VJ+IjK8- -zMs1KC%{g4Ov}uar-%&Ms*bW%3S<5k!QPoaqB)1o8{^PmG@S8?y(!k0g(UZ)e~0x@J+yeO#+41ZT@GkC3GGrF~=++)-lfc`QZ1 -zl|~~q2CIrPG}#QDHWw9PrPO^@!6s^~!H{R9G~kIod(C$ec=z7#E%JwMJa(lr^E(ME -zdy78btxk3jgvswMqwFmYF~)q!<|4*g2~DGWvs?A|k3J -z=c3L{y%l;ZDgKL*Cn{LPe}(lP)w?YGTb}uap80Qb=z&P! -zFOevb3eKcL@~KcQ6`oIoPnOin>GWZ^AFhInZT==A1iuGrzeFNb{oA>Y*>ZKV+S&Ke -zdUcgMSXZ`XJwD8KWPRgBUH|&c*F98_hahk%<_+$@>ZZaXAIc=c(A*2|iEfiMINS2f1d4K81X -z&fv06Aw#E-p;Ke%3^sH^2Aji|!5LiU6f!u43{H)~8EkMuhIXqjLsM{BlLlT_hgwh- -zLfFux3GWY8Z3!P4YHGb54u)^wQFANq29KH#q07z@&qeRdgImQBPesQBGJhmzk&s?i5YAk1o;V`x2s@kM(yT)Xn0Q)x!0&`PQIjSo7mgsC?&lqwHn5acQ -zq+XjgH6`C$FwP&&fR}?sTI55s<+vZLzH`JWPr&?x>@H`E`S>4e6O7{dn+fOhq($|C -z5^TPv!$`*ytAg{tk>nr0fcE6%-+Do(E04wwC)DawcPU#4=hH}pqw7?b-nlh;*i`Q) -z_>sfn`Nd5E@ebszCwG3oKvZVPLmS|s3#H;L3efzD%0 -z*IZuH185}h3@B+{EKq54#eWEAKe){Ft!~!HvJVN<$&qEnN#)1p8*;MRHw5pTUs7w% -zX#jb-^fCy4tx+zis2-l_iXz7ii%nj-jYo6=8-ng`qq&F0j9<8 -z1md!M7j(tZfy~DsvsL~k_p*>1jyv)kE=k(a5~oY`);Xmbjx^wT(?omT8hZvvSV^Fw -z0HJ0q*!~-dF(!%-RA5PXY>kwEts5-#VrQ+OU+Xq!+O~fcd}+9N<06b{oQsPs|C-M~ -zaSeGF`5hJn=F_=WVr+|_i6G2Qqq?fK$x4k)Tbq`wnyM}U(7B=grLV)1Qr78mQDa$R -zESYod8KsucK#R-6SIZKy?8X7PJL4lg -z_%ryNrL5>m06uwXwWu?2=|>{4TU0Hr6HSIEI99divF6X=WwL3%jqZcJ1ftJTFS8oa -z=b`!hT$PVLhrP^8bLEFX_h^T{kIAIQ*NZIU=SSj+)I@T(1Xpr%@2-SX96s@GqCGAi -z`bC^iO=hVHFvP7?EF5{U)Fh-x=eQmR5sSyxqD?_v&oh!Yr7mI;F0X}W*I-b{`5Qfu -z=+_31vv7BN($?0>)CFld1<_rwhXqeCZX(u0OiH_&=JimM+*E8qVl6JF7S7G5B2jtR -zQ~Vu-SqLoRp7#Z2!S%OwgTVmOAOsz-i5NX8kZ_HDtel)+#ngOb`5Oj -z98|0o5kP2r3(jW)8V+c8Yzt04t3Xu10R$qzSwC`>fFlHDMw(!RTO+-vw6}YAjV2KzjQ+6mK0M4q;MEGf`aUY1~ -z63p~*^Q=8?iZoHt_^9yL71iNUSamSRh%I~OI0({2M4Jy7?fe{bow>EnTs@_(>PS_J -zkKOeZbIOcahMXio81q9SH?TTL2{QWMZuQ!N)NZKqv51AK5veY-`AkFtKFdu^yg8-o -zs=14|x&LG623}rtlfYvk(2N%c;}P98qIy-OsJm7)S=lCceP>|J$C|}74y>#t#)0%1 -z!V(iLZ>0h70Q`m^YD;B6BvG%pS@of)kH05WK7|sr8i6p`3ow4l)?<-wFx0wVS1a>AiO? -z&!&Q+2!DxrK}ToEGwngJ{3y&bDyn4gRC7*2@}IaA)__yoxIaM^|LJeSmx^53le-SY -zy@KREJh1oT&5NaQOb#}gzmkl9QOVrR9k4Sr?d_@R;P)%uRKj9x-K#ki5sJ6cWZWpz6r)b(-GQ!o^VkiHfPz%>#%m0HrHI|j&vjSZzw>k>EIi? -zjReSe!J7^$Vax(bv4O|oJQoG|iGhH5JOoofG?F5*?*OyU?`b1CU<#WcIsYvfbhMrscp%Q?jqN+)Foem;Ziz7&hz92{nM$^yO8L8Ghqq -zKlOis-gwPBD{wHfdzYo>q64ZJ4a9Izp(#Dyn?+V4tft)hgCwoVttxV<3=Y-XR@GbR -z#JhRu#Jk(W5@Cn!d=}h}z(=3En6d}vKcwd-*6854TN^S?tH<(Te;$&zTAEiTtx~=& -z@wU0TCRC7dHHF<-UCIc=$T(B{8aG*zbYP -zL@UurY=|%q0!5rKV0Eyn}gY+W55kY!`e-}Y|1Iq6f@ZUpHKI({g}_W+46_Cb1sHul5OZceW3iXc}8XekG?B1O2`u%<(@=0pY`B4?>e -z=%0OmC-*CBS2t2D^KhifRF0eUL3}M((ls)*7AU}$hq3yR%*G^snA{?sI~yKMXVxc? -z@R2rWX;QHv(jasWu<~@uUqX84+_`(zN;)1YNbLP<>RdOMGie2e}KIw3o2iT -z4S)JcSNSBDegeH4B1Tv#&^In>39#Cq9?)EWbkzzw@$!ekj)7aQwAxoKRyypju=bb8 -z(_UP~o(M{$%3Su>aQr0b;2o@nA0>&L=`a`PpAqLDkE@sSBPE5m;XCn+B=JidNb4gc -zT%O_?CHI`dLzJ@Aej}mAL&BfO@j%dG!QD3nYX_B8oryYTq-^R7E^4d8nc$}E4ywI7 -zC#*gY820U+PNkc$LwN_$juw~zD^bT1Tdli5k=_6=y&m0)^n+g;iLWt#T{~K9M&Fng -z(Y4F0Zd$QfJ0DwE1xdyQn31SMy1~amBk;Al%0zu#(AcWlWqKOhU2i2ohtOcy90j@v -z>nV_9Y*ymf?auQc86Jyus;|0;x>!(kZ;gd*dtLMr@y74reMB$;1%WpZNu)pJGZLM?7~5TBzK2eL#R?r+@*4L)rBI~G1iI@I1S)ZU3kq6r$c -zX}ZQlw6&L*wUi6mJ^{7%Nw6@(78YxXuYnR;!Ud#I7encS`JemRz6aXwCJv*vU$vz{ -zgHh9Gf@J|}x)&PXiCSJ4y;ywwTd#*W;wBm$gvCX)sUWH3Y6*yEN1$acs?9-JawmCQ -z#QQTlO<;2aN{*`VtFXizb5Yy7>VrXM!RO$WKn^!$wVp~SYb|q7jSY2WEC2wWf~EuH -zBR~r))eJ-ts9HBj6#DNyZHOptq7EP;qToiNncai2k7Lcs@TXy!g&qYWU<#cL9gI&j -z5&wq9?wIz!(&X9yi6#?1kG*OnfH3M#)8oJ|&AV|2VKX#FH5{=4C7O#uF -zY>ovM`W(oT0%W0%xd3ybktM9qfvP%nQ;D+#cm-P1Xrku-9>Dl32`q1{zjtG -zP1sBzYrbnVT#a&5?}4R@k*YVeL^-H;Hysqq3CwUTYL>DykbKL7?9pJ#f*XlFELDvr -zT6zhD_Ms`0UksrdH065dD}EX9JaUcg5Ybly9E8)_A*zmi1rPL` -z3hF8ao!?0h^_83Y!bG%Ik4lXvzAC&6FKSSyaHXe1ga;Po&F;=owG7)n8QXV2Qjd2A -zrg_y@3zqcaFa@-Fh?5TD)HzAR2jCVH4;uJ -zgV_J{mI43(rJ*d41Rz#NL*|7>{%jU%NX*?yuvzel{b+6em*~hzG!g{*iwgc)hGe-OTMm-ORRX -z_|q_Up}G^*>K6tOm}-W4w;|+ra2fSd&xs%taT?M9sH;O{JtGD*00HZ%IKagHwZ?#4 -zk?heCoc0hc>YNO(Ee*!w -z>D+^TEaH(b-GtLcoNy2)O~ffHaU4zcr&^mXcV32Z`lZl2$IVr?+y$5LkoXfpnM4Py -zUcLnj)-W#iqR=QE8Fi8W+eLw$w+^(Ewg&(^N8nRPrUp$cUsY@(nW~l>oGH}{k -zd@^v_e3p6}IF0}9ii23dQU>5O^0zB4Vj)Wv0jJTPD{dl;h -zi6X>l#z~jCIUXxvp0%*yFu~9E)XVrQ{x}r)#DO?x6)c*(S79W|GYP1;%8+Bsq+S6< -z5mChwFYX}48{F(2JTt9sh8l~2DHKU*GyEMk*ZMN -z^-v!d-UP;WwI#TygN~CL2lbih-N;(EcMPL`z -z)g{ec0`q&F=R@oUDVEQ{kg;UC2oLZOAG?WzMK8@lTzSYy9W{Q4Ird;^wb(Q%)I=O@ -zaGK+s<`#U%0hx@`wCcv7~lfh+g*6qO_;0% -zvk=tLufov}#A0J~n1g8tX9=ya?PeiAfAyAwU|H&4kP$v14`1w$0tDkA{)GVje9KL| -z<|1BaiFFR*Q!8=Ex(KULsybcIV*J@FEVb53du=E -z^_iRcRKDL!%LZ7CzthDj1Qz2923U;T6~K=LHrh#Rd~CD}O<1RWY_uCqSYP|t=mRui -zwfWd+51O#r0N0%kVwa29?Iu1zY_w-mKN;cS?sc)$79Xqfe{^x;i!8O-N*sXm?g9S& -z8}RQo;NNB?aND94d^oJ(#~j2<(AV-0Fe_2Q65D_paJ%`Af~B5CmIL-kkQ!kepL)$F -zID*3-mimX4qF68yAYO8pJ6K>B{c$J;X1zBdaT&6Mu}?z9#MV<~1M+4!q&yEsFz)iN -zA>MV$?AE)4^ZEgkogT%dh=&}6*+snKHZSYybe)Hxz3t4T2oJI4Lb{RI?7RsUnyRYY -zlb|~L2V%cf7~q93#sdpsw66q7(nWpfIQ5Ey`V_{j6}o`<6qv_o_8Q)`g%<}n2)pe+ -zW;ZesPUz|^vwi@#cZ6eAE4*gbWVTW}5DV5K&N^b5}&HP -z>f*F2WUFB}QJ)#9Pd(I!x#`!e#3wFdrH=^)TG9k0`h@&jj66$z-hWwoMa`J~Ha?{#2(VOr&St`Evp -zTq9_2_9D58KMbLdkX%Kh5QpR{?ohW+t|AY0`{XL_vu>YUMSj+eoY&+jBah*uUYb -zx$i1$Ai#>R%)JM5Q?Emen=-|r-*r8K2CuFAZ{p(z?^TZp_7I;Sdpjcor#>F)la#af -zU{b9fE8bWKPj;H9ij6h*kY1llfO%mo$aQnOLMYAZPZGhs<|m1Xgywm;0M{(%le(Mb -zwNkJKkdVLo!R-QYmGZNGeLgMr2jPZwQQ5qIbqN1ft7^kOGI%D@924=4UY^HWqkmi+>!`y;k+ag88jDWsly-bM;~dxz6g+lrC0rGV#QpA~f|F(H?n~px -z_I+;Fd~AMA)}mZ{KvgWSRGpN)uB?)8R-u~~XLj3Y) -z2)NC+;kU~hWCLzVklWfc_^`OP^^OFYhnnLsp9a^04)brq?;ZHHCP>4_$-@7Tdc^ac -z>2!8a^cH_6e+K``B_x@~aMOC~MGo_At4MAXDNJVA)_|A`rQ~oE$e5rY8}8JLNNYb= -z-`@=F6)u((bK%fqXC1lq{XL1f&-Sy#Ebi#{(4p0;-M~`dwWmf^FE$c0ukWAhdA^+c -z&O=-A@O7ZI1dBF3d9dFh`W~!eYz`@$p<1aq;ityBMBj~H8H9&NzU!u@foU=>iImIX -zWeIsC?A52iB@qc37w8auXB0_IqHhhxu(D6XY<6kc_5>uD9Ie<<8FFanFgNj-iu1CJ@A1ZmAgHbK;*a3@I4j&dc4E2o=4Gd?POlpDII -zvY)T$-?+g$)ZWp7s-4c1va+z_(8o`em5np+S`QSk-3kT*y6vkBhV1^UHfxwi)B&iO -ze+UMawK;=b=F^xr>efT;#X%lKtacl)ae~w#`o<`7!8zr*Ibe5xG7xM+Hji}CW-Lj -z_^6bRytqQXR~UETS8>ghq6EU2kRrlRpeO-eRpFG#fd{`A%vkVIr6*xB5|Lpre>98u -z;6Rvs6>{)y0{Vab+JyNsB64nZ*a+LQ;1zT+IscK;38g>BjLZBg2t4s_#4sXkE))Rf -zMfsT*6L8h6MSPFQ_f0KNcwBCKHCUaq?q3r}On+=a>J#WTrEuqs%7e7(;lm(#-M@;% -z$B*;_o8h(F;Zn}@=)pYtrBFXEt_}+ms#hB;NvODNLdEj;J%}AqwWwZnhj?*9oYp)6 -zVo)sD86_-Cm^O}f%MOPXOG{tGhr>0aLx)AUjCSKIm8+M}sTaTIM?Ql6TE~j9+M^9P-Qe%y3gTnj2}g%%;U)XLxjXOfXYg2Sf|47L;ZwnhI1!#u -z96v#fN5avAW>N+JVt?)oG?)Cd-xQ3&kwRrf3fR;s*jI>?|uha61og-9Cy^Y9-m8<|V!`z0mWDx31N -zuJ~g4<~f$1b!%Lb5_fOMniHk9ss%F_#8#nS>kwh8JK;Z$FgovrzcvC_ -z?C!0+nDD)>dQ@`FxR&J7b)`=uChfwp#4m#$+jd#5!pgtab+qX!70m(Fqrz;x!IwL4 -zs*7V!d??Os0jt-bGrO+g_)sYp7Q_SAGa$3y;f{%(f5c(F=e9?>g8k>qTuZR_vk9&x -zBH>po{7S5%NSzs`RODF}M(5$-@N7gwgS5}*^2E+2L{_dg>HPsLnbXf@^fNU&aGs8< -zmT!~dW6Ymd9G&^Ro!P5{Ls5KeUvk_68P^#A0Kt8aNu)6)jBfZ0K?bi&X7e%#0KDSW -z#-(p@*?4%Jl(lO)eZR*($sHW*PRF`>65Qz`;a4pD0(O&JqRFlVO@kdC;|07Mhw;lZEuCaltdv96TMQ$98$wDgOZsc=O^Or8!BkuWXF_mER;^7nT@e669%R!(xA34l-*922(zLHMo+*plV9R}NSop$WiTg@H6qws@ -z75ObpZ1_ZK&3>B|`I$c&NAunMPuO{~{z&KG*;*v}vRXyg2qR~J8wW| -z+;7FBiK-n@xY+=PS{!|O(TF)oxX70}BNOsdn1YzEA2p+RJbG+^&N^v&OVHD2gSsvT -zL#wVO!EyBzuWCz~F*|ADi0(t`?(cKDu15~_bahAeY?N=c0yt;}cv{`A9{#F4>RolK -z0{}kMQ=s-=v3A|6B&PN67IH(1c&KR{TcvOM?#FsZ8!HX1|*`pK%Stlw7N7cRh -zgXa<0N2^>A^}{2hj~9sB6X+Vy4p}b-tXVu29c;lehLr7GfI37MTk_ -zzW0%aU=7#zh8mn-!wgNQtLEUr7s;KZ!?uUBKw%IRoHMSi-1c_)m&K*dQTx8kJTHJn -z`%nOj_Isi1ZPDH+uFzM#3BmZ*Ett -zmzkYvU2d?ZQXQ1Dawd4sj*gkv{t@uW?{r*$BE*)QwKbO0VchDi3E6(pN0-%>>! -zb?qC`(B|Jz|5ihrrgoaHW?xe6G(Gz@uwOrvhr-<7-N{_zU%AVM?kn&&x&4i>*@OXd -z`v>?O{$9tf0mbvX{reX9W5*P{j;YEO_y^|tbBY^m$BpOZ4^<66YVzlI*LOA4U(I2p -zC)Z1Zywqpk^R?4d3B?715yIYxOjFmrXYE2;xCNmTSp%mC)g6^W?2vQvT -z=ete)YfY9MypOUm0>8DUewC(B#SMa3{ybXb-ZelfQlYHw?@l-L)pqvPKuZ!F9DY0d -zs__^L+^6Tm*VPtr>O(&FyYo=L>}KkhT}; -z*{e6XUu{V}0;17%Lq2x{>VBo>hovv@U<Z>I)!xm|D9YsK?; -z1l8>-kk8=@`dg}+GhcpHc6}X$UF}DH*tDClu&xLYzih?yPp^&5EDNZbkNde>qcwY@ -zRPP<||0GSbGEyZM`;}uK(OPdTfAbTIwc5x9kU~Kyq?}J5{L=4&&RXn9J-_Yk$)T|1 -zf(!9~Jhku5CB_TtpcdEl8O?#8{ATEp=eY;YByr(YNz&p|RY}dy2OWTL^Md-@3T+!W -zt|QM|cU+&bEotjVNB6zC=38~Fbz>{}Yc+dvG3UPB(H9*vPx>Bu`rJJ33F!yAeT+VM -zQ?*1>p&v6MHF8d{==2+Alo!aaqa#2h=bu`pmJjzVN^*@$dR_a8-42Qml>+R=pE@P} -z*yGxn^xSJNjM||cZ<=PXIu;b1*o)EDElOILB!1XhZJa#ehz -zjT=S5nQvX^U~O=6hex~(uisgXMb^sXY+C#<1~IT9t6qM9bDl*v{MjU@$`0_@;d1*` -z4*aNe1d12@%Yj^sI0s9exUCyZZoMaS(iJXbC2Sg=pjje)PRL;8Z1QGw37kXauMi@2 -zUdEd9rb?P~ihAcg>K&acB~MCkgKi -zZHti`Z>)_+C;Fu~7;$wat3Kg=tC$(I1ixev3v!pmRl}! -zq0+@0{TDa!mu`H!_=lxg^)O1HN#SDo8Hnc>1;WmPLDplOCDTP{VL`0+XsPBQ`IZ)n -zTv*A@K|q+mZTh8%K`-%1Oq`aX-;Ac}=U>g$JrqV#?J -z41Fnf-lMIlNqis5MDc+XPdNhp41PpMlz1$)G31m?6b}b)@_Te6=*U01Kniq^7IJtLj=jcAZSBxPzh`!i$&dBsNq9WgQj^P50NKqCkptO9~6Fjjy_2P@wU-< -z*7*$v2D&!zi@SgmnEGlMH8OBd9}0#GvkjL*j1wl%;W<}i!kw!3t97b7r9E|1cUU(% -z(mJQu@X8$&sPj<*;hDGwcvkm2`KHdD73y~Fzx?h<0_^mN=14H<*hOGf++3B^wIjQ$ -zF1xEOyX#^$|28Al1SdpXIV<})2E@p}IR$3JI^I|d>#@d~&+_6v$iwPg0<|4+iARb% -zg&kW4NTmGF%Rm#mhvsn7Ya_E7bm@EZv-Ym3#2xdo+`ih(^QPwH%n$M6s8j@0lyZ%k -zXN}U|O%^;Rpq89BED3&&kRRN^8kg&xDeIpyeJoeqsL;z7O$BO$-=d(>FGhklM8#kU -zv3>PZ^0RSTVsdj~w9;jra4xDa@GEZuHG#tCOci?Rts8Q|4L;`v -z7iDPiSeAQCX@+JGO6S7+pYiiMqZ@Sw-E3*Os=#1LyJ||~LMB7AMw>Kcc<#FMV=s^(A75R_Qn1%F|pQ?#;1@DT8vNa^())gf%sTN5hzdlK;&~~80@m!-+10K{^KJgmmOWt!!$$0ur&$zd5kGnr3>Orz$gNh-Z&6x@|M67$KW?faG7;?!5l|7uSV9#@U>&aASR`pR_{JOM_CU(`-!{iIyT_7xbo@9N$ZIV&9bkoljK?a84mV4mFH3xE|Digo?u(am1;up22dSzT#hU)$E -zVNH3#X&ql0l*9GFMz2_WRLny^N~UhH?x(eGaBUk8 -zqko5rUtXr&I_<5q&Xzwf{o(91qnzLQx>3&U9B-VhfpxnjM%b!r^Q$(Tem+%QZ@jH_ -zyRxO_3%WLKJ+AG!T?@pCU50_GB8{V1BZLgC -z(m2Lygs7-}8b_?&k){zQ4rtamhUp!1H9|(m07$@#pWYS039%uNqWo7JQ(u#aqIy -zeNfSdI@1ZuZ~I)=I3Cf{yX5(SMH)x6zavdwx%-yoGmgOWcg)qpXt$-_bXwyW?(ay{ -zcca!z#(a%qh`%F7Pxd-n>qdDCJzMLa;-lXBX9OEOR9o7TW%otfT*H%)+E1A>EBk;ztIe%u&hJ= -z(p;o@y1d3>?5>1G$j>gKh+5~-p!K74&8}8zX(>zv)N$vnEG)h$9NuOI&~scD1=a;za`h -z2JdU-IPo9;+;Z^^)(T?Ulm@(o2B6&h^!HyKh=xB-0M1kZTudO~ssv1aD*#Hf2*7kD -zcA*l>$=6scD`VBdf#7E|P=@>%OC>od6;^w^+o_Zx&OjORG$r;KC6>cHpjcMMR+O=r -zNA~s7RF+E2qEukC#LM0?cw{$9sY>kAN-QT{x?9E&%_|Eh$(CP_#-?%%#-DVX646kn -zKgM^FpOiwtPGUvL^O_NM*1#>7|9Fzig&-HENsQuT0ytR#cuq6io4tcCJ*V`Ya|3cL -z>g`uEZ1!m6KlS$OLK~mj$)kWo@wW=dvsklXD%{&86R0`klo~QvbEF;?r>q+#3BXnp -zP@(A<2y(W9hB%f2=3`VFk0~Q5Eg)1Re8^A#b|GW3c%=sFJy>d|%2+&u_Yc2IPn_0kM$>Lftg!)9rar*E0rzwjJe7AFoF60m -ztX`-T?3{a<3(MT5KOO1#ZC^(e-ebqj7oB_%Z9(`L-StSvBYhnq6mG58iQlvE^V{+x -z9i#g?vSCOS5hl5w1-GX1B@m;EC7uf~1XFM12D)8Y{P{~X7x{5m4V7->Gf?H;??Ie+ -zBec}e>x=y-H$%)%Zho=9VK2^i-c&Eq3;~gj77`}q?!IP8=~uA(XUif^$QGn`VX2gV -z&r_`@x_&{2N@N)iFXD0{^a!{>=?ohPpI4>_A`z>r+%;YL2m*w9FyomrCw}B -ztn;7Tc4=TIwmT1Yf&m2b+6bE;=%n=5L7l8Pu02Swq>}n -zd(TbF_W`tSEerrK(N}#igq1}Ki5f>5gGFL}4_L&(1k3)*zEltPVnIKGiiTTm^1>|* -zqO<{*#9Y83=Mo&%UO1!%z)`Pws -zi01M^Xzt#FG;WY@4AF?D&L@pxAeL=j*NI(@AJ%Shxu-lyTJEVz8l!I0U+}}qjiH;* -zAGkE7-wfwJ>v8*tLDGR*EI0uQ8)6Vbf`q6*VWLsU@DmbrLN@5V^9-&;Vx+l&QX5iE -z)~lo!2_Ie}q|-x3gWJkz-p)i1}S{0%bqQLJv4~TP5 -z)-4z3ove!)22K(S-~@kgBv$Ex!;o4(_LFV9xbt#>nRA;1El{|KAvBd9@pqd!UTWi7 -zr0thiYE=CfYxvtIxwF!?%ZRAkSZNNdn@9L=(5aLaZqP|?u{4loUlxd$9mK#9z7GJ> -z1W5G9qe`GFzBLH$n(p*|`xNHjFV9QoqZx -z;dot(6rv__DutrsX>Xz7{f|Rvw?@9%Cdq+SwYQW~?)jH%p2*Ix^VUtxA`hR -z1)MVW#Sr}YuMWm@)OqB`T9E5}6{kcxeii)jSwYEJ!S{tr?;w0kC!H)PYlp@;8VK+y -zLYZIT<)tM*c-LpeFV6A#;})ny^$B@fTV0}LX>=~edfBg0oO-fC6UR1xvbuYiQL4No -zZEcdDIxKgzIn(6BXT(fjef5~r@wKs_wmmfcjeLKc=E!}hw^{vm`v7m>C||oh%{xW+Y3gOzFSYf{xBrV1Epg`P2C8~Pw3VO?%Pti8~(YNRpenVP4?+^3&OT5Pvx -zL{j{Gl`zp?$kq#a8b@Io`by2OmjTb2wqeGq;4*c+MtZ>Tq|p&-$`3Z>uiDO8=k8U* -z#I5e@7`18caCODVp -zYs%ECwyVowWOYg*ZNC_??AU~Kc^u&}xF+k~iF1`uG|zwy -zmFlh1oV=i%f98qj+UWY}DTr+3Hjws{ZLEL5n(9@&(CEY|l62l7UvX46=V!MxnHw) -zu*xzQf?4T=2e(qceo{M)wvF8=Kqn;Igsdy2Qf4nE*9oQU~%}g -zuHoItQ`P;IFce_WV`?JaCU<%m$4hR$(=!#XbXn~S3M50I3YfX+08;6sZclgNunMi! -z)15I)YC$hpatnujvM;EzFxGttI_d97`bM9KZ%cf_d-PSIt0s?;HQ3(&3E^r -z5cKu#MIB}sDw94p$o(2++A>pG-`<&+HmO~s@NRqu9MWP8o3!RQW=;++Du|Wj`X_$m -z?etT9T>3Yji4{xXk^GM$#K<=2D6^op2cv3SSC+?L?v&3pI>P-NFti@d7r{Ip;)VDi -zAw5t?CP_F~&B+f!gM_m|X##Py`xQTLUCvcq&O}}6#2XoBvX8v5KW|%M@s^yNL6o)k -zT{#Ydc1~|9UN<{W`cB71;b_C;nuo_3x#+#DKl`ao2R_u*6y`POItKcQ(~rAG=6$GD -z9^C_c;@k3gBfs)^$wZ+rlsmnuFmRKxOe4R=4%MV}_*mw8} -z@0b+XEjRpg_xD(e{M~q1BGKP5W)$k}@R+t|4h8lt%`ph^&_?StS|l&^UE#pGaUD?(iUmB@Xf_FpjoYJDx*TVR^hX8-&)6Tx0|dZ -zxr?W)zfp9ntgUCe5nHtCqyFN~Ai@p@#2n -zTbB57|2k!Tp0j4g+;Q%_eyHvZ5X))D!NP`T8gCbM*QMrPsY`J#D!Nh^4L&zKbNkJr -z&bl%CP6#$XM;O<90(gHqyKXp(>$#DC<$5<1qcb15w(n3;Lk+I{6#sJiL~|$fSu$_` -zd_6n!i=Qbat~KUoXXamLZdkkqUYD7re|hTGNH2HRC1QU9)DCbiicRC^9pnFk9+7^g -zpI^`^NDInJxQH;Nk1)L3i8B6O2aaZEk)8WM7_%f7_;KCOnJO&OIF>zB)=&VeL^t}F -zS}fE0ZiZ3t;lbF4eY+k{&|~EHbegVYJ_ai{KFyLcc81 -z2+6^~IKPY*=?xb@LDzLf}SN6r%X|j`dXD1eWxJVH4 -ztR{105C7_M`PJhqPkO|!k7eTVlbdasXRi~i(}b`zN+^HXY01#T^_u(*oA${0xqfns -zQvB9VM+mE&-_^OY(j&dxh`GMw+&&mMy^}fa92LN$&P@y~j^m%vWM1u-pF1u+e|+VB -zR)n;r)9@`eAQ)S`p(3QyGe3-Z2koOUJY0&Wf3ofk8)MGD0u-5DMRQ|)o^uR_glIA> -z?(JwSLsZ?|W^gkHOA4KHfFEjA*_CpQmqzhX$I9*ed&kR@<%DD9${JH<7aurQ7SE5Q -zl^i@MPQcaD6fHWsLd~aW>lX88I$C_TtTlMG!gPLg0$eHP -zYHisd=?ARid|_jSPRjQ0OLA -z-_(f)enSmJnPpUFj9Ax*`F9wJ_#Y9=|Fsdx$MU5tmsmib*Z2+eG4X$+sp?Oh&n|Bn -z{79XQ6X&D@U+7I)FG4&sfGhTekP@ciyWV{e@U69M5m#%1DhtyIUrdxuKd`?40 -z&*E7P1plZL2L!8=#ETj!g+xUup4++$`ZmmvdgCIs+qJbQsW^C9B>%-lU)#}7J(!PT -zk9V93cHgq4*kaa|6-t_8j(DRqqopoQ$kquV;MVwK8l0bz=wzJa?8FfW2774D_ClN@ -zJR!f`DbGG`S#w*7=YOy8!8oLvif6q73~)TlZIX=;eiC}HU_RgH@P$Ze$VPaLZQ9KZFs(OLV*Sz749VL(BoD7jiQV2Z9_^6H$@(K4_nR=as=Vz&o!bbY -zt-f|98hXzHd2;{k&1d94>5#ray^Mw~#nBz_?hrrifS?5@QMXAJr>TbkICeSjB#fbP -zxk;<;P3%W(PtpX4O%&H$beIGA+K$%6Hqa0G2s(iOfExZ`2LeoTvyXQd3c^PF2;d!Dp(fE?mS8F`4z59HisYsnZFDAf4qbVCJW)cFq-NdVsu` -zlsy1U3WTY(Hkd4TyEe%=cxZx8Q}KPU2r?4R@H4p!r!UD -zgs{06mU#qv5Yl*6W{WPqL7i*2z+SRrfL)&E!TAe>lvhz5@29KRp(efC^_jRFHC1cn -z6cx=>DNSV?vTD7BDiR-7aao{e_x#)Kq0|fo&-i$iGlrfBQ$o=;QKQ>57vPKoIFc6l -z>V>H!73EpQ8BIuE;7v~s`lqtu?V-{g5S>kn#WOftmCpcBG%ik6uLqO{x2s(K^F`64 -z1?%6>HJgby`k@*gJi0Al^O!vBDZ+J}Z93M@{Rp>voHJ43P%%bD(o_Xr)B$ZDydZ=G -z@v2sIsuaumP#UiEVf@`bbQX&t26t8%H*s$O_4!a356Oxad6n3X-jmybU~;#%UZV`w -z*SzDRT7K=wS!VK%p|}2@EW@0u~Qv^vd;;?V)9MNdT@5K08EepbUq4Gl|ItD#~+i6KA$NL-<+k&QLy+rnPrWXXm1m -zEvN;6F2;QjM?c&k*?B{{tw-O=&vc0G+LG-Ujxbe8n$)gU5S{PqLv*nZl-sGR_jw@l -zW2fg1@v)0sw$cgcYrrG^=O=D}tE^DGM8ZrjE?w)`xmd3IX_`^Ia2$1ybgV(e&TW|0 -zpOFwD!m}5zMg`Q36qIn@=8-pgfUV(TIU7bQM~G)kT`l%jw>#?Q%75u#3?BRnKIM0F -zAwr6&tO5586Ls&0gQF4gW{AlJ&FXMdR74Cd{3r+QyX`4S;{tN+FOJm@5gULk4f7=zuG{*h#G9`b+&iU1U{IjY|_7!B9^_#p? -zTRxFU5>_^!ECF(pkmwA~9L?6&H&w4=o2)NS!UX}pYVB;T9NaX__;~nesYzdLEK8D# -zHA{?~ZQM}nq^)1+s1rELqqpV5dcbp7e_MaS(58b>-Bt5*G`Nd@O3}T1g>*>YyXMFZ -zaC_GrObJsgCp+(O31hOrP@Cn0`tsy4OMQJ?WEaL5gjBZLOLIG15njGdt|VsAyPBhI -z5!p4P;tlpj&Sb|!o`#jTz5Fs78`X}k;C*W&c5%`Wc#wy>Vz!>gc<#&jAJ?ix@7^Ms -zs-XVj^P0+`(AHG074f6vbLpUi`f=}F_<3@B^=ia#^qje9Vs7abaE$MdRz@D_-60x2 -z*f7oL4pB=%kj1z|uw)sMGgS5rwL1f71U@d;+y^I7xG-30<|^l?+;h~qb2LyO0ByVu -z?y_O`OWo$cPBT|;L`vJ-)oM8eq=l3_Zg^n+&}~yy+SIi+jr4x27oX+srt*f~cJlsU -zZvo3xH?GS$<(KYY(&`bU%qPa3h#$0Kx|XzpfB6tD{vYJ}h+M8#lK#rc`g&bCcz>W5 -z4)_;x0mV3N@%9zMXe|XaN|_X_l+Al6h}<9+iF%ajM8VP;6fEbr$^}%*BW*$gBv{;7 -zX*QS+>GyiTI$y=5;VA3z!-yZ>1}LV{cHw_0+e-RbfsEp+R=GkCF-L_yV)NAo*E8Jj -zxxT{xoNS|VC^An0DGKFjJq((aa6McCCYu|h{KjqNu6V*6Nk&id+jLE-J5EV3~Nb`8u -zkO{7<({a}HLTl=k^z0zaS&-mgJeV)I(o_$I3O7#Gwz={zFn)%16Z1gt9;`hM*=l7} -zv^$EgjS4S_ip*EzHfX+oO^wdw3WchC&5q!XjlmP@g6T?ER9H?90(EiBsxdChRBNGD -zS$wE0vM8_2xNe;;h&`zdjc`|JQ9#YcV3gItZfsO&pozBngnY$2cKni7pxu(zCqXLN -zb25jUoEvkaf(xR|Q;Kwt7pn($C1(exXM@sWT(tL#kJWX#4Q-Rqxx-qKv$WKi^5G0 -z>Kd~9ap7QyKuu;VWu<6m!=&PbSvAvV7jDv5zz}XL4n(lZ9HG77`0Lu!AsASHnyKQE -z3n|*v5NQ1eSl;4<(5IKWHsj*DWn&;To-Jejg-L<6&=vTj(eaF7Q3llxRtufjj0YQ? -zi6;l)hAsR$yop;kt1hu9W3n1j??oBhGpSl3tSD&(Nj -zU!Dhm8!;sOqpCXMKCIwL5i}Ji2_K-x&Ojhq$PL2Vb58vtq{Rj86Q1Q;km9^Dj><=F -z^i*vy?Z|xX7jDwZW>0b9qNu&F#KukxNc_QE5#A^G&(-76aq5eW+Jia&?wBrgR -z=L#)iM;Uyr7tg`_CX8W+#tOcnrd}BiEiTbf(Fr(H$iu0^20BxS-W`1+`cU*0dv=gL -z+icGcwr7XfvqM*FSBI?5Ui|{YH=Es-Z!?XHwjJO~U>MkMnRZ#ovZ!SV`)4kjwCtH> -zx@Ciwg&ukF$jH@V@$brKEz>XKm-XBB(zb_Izr6b4Ws{dVmKl}_%Pi5EG_?rkAz04n -zyDVYIo2eV-oXE;HM@La4!h}Hsg^}U754%O+!-a=nWFYm5AoLd=4&`*o*&zeFQnpwp -z;bw2j73-vx*FDKmA=XI-I^;***ro{)Gdt|TS+#Yqd_5*EXC}9I)x#sEubUa`_x`Bp -z2-l12Hr8w``+Cg-)6*m4?2)#-h_7n4|8r|XA8u33;g%l{9>V3dnN2%;#c`w~k{e}t -z^-9?b1yTKrBg@sxg3Ct~CpiWTfF+0#^?s(*Hl29gV_K6rn@dgdN=VAGv0V=b8?!Gs -zMco`}nuG)QUntE&W9kSl6qdw`PT(!lXr!o8FN8k6$x&~>ehVpz{9iY47NjVMvN)n) -zoVZFoJ8~cHeYffPZs4{5U-hfK)$m2xEQL~u6_wy9l2YhkMVuKK7*uq^5!PFY6_v;r -zkitM!(1d(=*_G6Zi*~1Hhvp|0jVOL0H@iP(+CwB<9Z5zBg}rTFeghI!TUkhvwm1YG -z=-O0RGRJhpQq2ittiqhBjzzlD2^}Bg6@%Hb%r$JAxNSEm3~ZZUpIElr`9^+r-R%8D -z0QP5qIysj&pQw9s{~_`pD)~oul*Jv@ukKPUks^rb@}qJ-$itOugIdLcENgno4p|a= -z_GMVk-`JM1?NB8W2CV$a(;QWn;20Rj&Edv&Mum4rne#WSe&NQI&g}kFzo8xN4C59x -zL|HqdLauMA<|y>W3umr0WcT0O1Q~r@e%Xn-w8|(bSX~8&d63Oa9#6`Sf$qG5qq{ks -zGbCbjfHgF3P0bWsiU)OOc^x6p(-)K4 -z(Va$~KEO@>x9#P-`LXS8ES2N%%Qnaemq)kv%4g(P+S|a6fUNAjJHNrCjM1c;#VTDn -zO&h{*? -z{~yJ&`}`}!Pw(@Wh@U8a2F)!JAt*YlUU+VaAN%4Z$moWyXwF#^w0V03RAH*}5W-;R -zbW_sFY`adpjo12Mvf11B7}E~cS!f@p9hDA^3F$-=y|&wpe7mszLo5Rov=9BFkLSW* -zyJqzdOCxPrk=FFc@Z`v__{agujxoE_)B#@L{`@U>0 -zJ^l9eH;aBKo*{NLk;aGu>k2t;0q0vp2>JuY{g -zbhQ^B&;=UN(k7Ph!D|5`;SScb?byC4{b!W7HBDpB5Ajt^ -zcjXV5A18?e|A~I&R&EXda5H%AQB(x2#dW!W&IDVZdvz*@84oMSbc%>W&J(niUwb~vhHrVqfl-H -z&x~v+<>JW&pgLVBlO9)KL|r}Nyk56an(K2%&}NZf35iJ3aT?_Uh~iJa*Do$=?xic} -zvqirAL|)J|1bSA#o?5kOR4!XwtQY90W2i!CQnRVP$n@xPBna1zDcl-!N&~lv8eXBt)Wcg6zfi(7yxtY5wU%H|>9hFSe$-Erq~zW%tA -zDN-n;nd;!PDEz7a3di8KHj78@j;}T13;pJeaniZw_p)eKT>y|k4EoR<`A}1=9_GjS -z;k?j~_cT}dRjDP_5j4n;*BrqX%1_cB0rvZe`Xkk_$1fX>&>C=1>F{*4@HT0Eo9v2ZY(#!f_=_I^{B&MLnJp*5!eFgJFJMHoVYUf-K -zWjjy_3EDjF85(XQzpe$Z8_m#tdb&@MdhntEc|^1GY+2;DXK_}AOE})-V6AAMAnj?! -zPTq0?$6Gf4SW`q*9<7Te$ka{PxXjQb&oCsD#C7kUc5bp42ZnDJ(gOjD*{hg6v75$w>o}lWcC+)0M$0%HWz` -z8LZ*SkyRP2lN#a3t_(In${!nE8SF0|_Qb|i1{-;N6Dxx`=|@j&T4it_{zvpImsaV@ -zF*LG0s@rwFvQ$^Ep&J|+Yg?lHMhiD>#ZGirmD-jF115O7Dq?5eHudJsSEXErR@A5N -z+Ua5);8medqS4<Dd6aBo;~@-b)KOxFy1_q;@X==Zg`=*w -zl3WoZbL?u(v(1+@l%25iw{O8J-R01em^mrn0Jnxg9GdQtc+`;*Pq*V#V5JGUq8AmCcF%?Mv -zR93@LoQXG}Nly$|RcLUAJGQHd{ikyIwxjZ~qYCL`ZTE(ARdMdatJVj^xvz>gKT;>L -zZF_&cKfIFN53(pI8$!H`&O{maq^;J-RzD>zWrZQ52jN9xdj?o0wuT|B$5W<;o;%*} -z*d3?(XX)lGL1g`DV+39~RD*T$^(nF~HLEap@j{3Qb>_I7SAw3-4tmZJftajxFAp}K -z>95FFJww#cyN|O6MMS)yGcX8O3muuydlWtdK;z`P|HRM{1z@J -zctL?8OE1jwMbCSS3{3{#nlND<0fnjLcDyp -zAl$g5LRmNxD*f$+zXSQu6==loN+(PVT5%^N2cE!b+jVS~AOTo$wKc?{ -zBLzs4Y(+Ts2|cRuMnM}is(^_*yYhlp75H&>MRm2KF!+6hCu4qZ3Fg*puB=(ny*UQd -zUzx{7zOBVIsFxRj9DOzk)9297T-pfC^UBPz1{g^PA7)zUu{QoX1nwkhoey -zZ}Re+TgxJ=M({UUp+Fu^Au`e+M7-E5e@Clh+tBLYc=hAq7swy&^+QfPP_npE^u58ja-lPuu2s&0hek}wG9#)1pkLrw|nULg39Orgl!75 -zyTS1rG>Svo#2oU%OSEG-F8ZBtga@)zR{8I27kY)8hhjMkto&fG&NtGImVfi8Gm+zQ -zj1Ss>0(Hwj;RkQT?5EFIKFmW!*r3IGhl;&|_i%LJes$tRZ;>(BPg3*bP-@;hejRJ2 -zxUZ23I#i#Og4ISFi9&21DaPS`f;g#`an}8Ai2$HWsnNGfr#9~KBW$9HI8?sF0YD?F_u!QA&V1sMl!Im1! -zU;R5>t`2Cf{>cy7P0|9&ewB(c$!F;^RMsmEJ;wsYi%RWQ`59KNc$y^=&(LS8be=wA -zB@NYo$(vHlLYeYKmQFs&(($b5vAhZ*2davH+Ut;>QcA`68*z<-$`ntrGR51<=eSa; -zNqz#QO03$ktd~%2J0*Ed{K=a{n#&5KV2C7GoYcumk*+JBR-|OOEP^!5mFaegLNCP$ -zwDf1|-|7}DqLwOg95VtpSb8+Pb0rU6^jn0{Hka`k6pULP-LWzkJMi&xE -zkOnA&&&FL$r1BMVPIGBz7Vu-9yrZ#N!^AT{R?!a>3N$+#=NjmtQGpmdNlOq#_`5YX-s|?a}+l{j&e(Biv(fy<_)>xgQw9%SHzbJBg;5s -zHZNHE!AF6{>57sp`W7weZ5ums`=-&EM_r7Gw-PYWnc#^=hVo6m`PDxg@8?0>idDiY -z?_*TFYj8HyY{iO3zicb=43$R`qEBV_gP;-aCM+K+Z)hBqO_p*@VzsX)vsQAW6tscLki(9);*uo)7ofHH2R5lh -zQAzxquP#xo;x}uc4e+IgBz96An=7SC{(j)~p(4{m6&j^wEUWYflqS6bx7a}#Y~lwI -zG`I~jP0K9>cbQr^p{kHyOo^vTv*=+WMj!8xYq&@d~S+Q;bXhWKh4s~Pd846 -zZCtD&wyDx8Lg&x;e0*$2W%k)2u7>YXpb9?Ytl}DcV%PF(Dam&d`%LAXcW4RQ!HUhG -zVqfE_65K6$y0J9_j-ht1otRT9AaEKBWu=;oVsypU2&h)ed#sP%{e1i2f~7T+u)CmKdDYu -z-Z=zFReG#O_+W0)N6TMs#FK=udrx4t6FF(@rJD@DZ?xbFLl3_nFo<6%_~BazSp2n? -zlJ!Dz0zofML?YWwc?!XMlb_rOb$gPbjr)%O6R6yy&q{yB2VCAujM&MJ_yG)Eue6l( -zC_I4ZQMwC9MCm4KyEGPM$+J)|e6QgWEqpVTzWq*qjt$zCUYwOKwm0Uzm6a+{=FA8$uK -z`As(3{1}RxDv$Kd318pv55dR@+YzwY^ivCMpi!4Nya{Tu&p>J=#x_Od1vv(21f;{h -zmhFTYJkns$M<`#hNj3Q{t;B9*tURNs6_#kwq8Uxl1y2O(OedXGY?l?nggQXbG}x?w -zl!Jz8a6w^!R|%WNNso8P(U%yJQv8`6a^$66{_@{MLL(K>UGJer?i3q0L=WhH|qz5SIZ6$wN2O40a -z^Q&+KvM@fiV;Zd+w4hb!OofzST~(fJy+qu&6v{qwKfnUtO(a2E16iQhmj&XSfLacH -zF#2GkEx}ni3%LqHr?e-z1M8?&G+x4@W0g&p+#60yOFJl$h_n=jvha}ty|6kD2_YIs -zR3O_g{fu(Z*qe-Qf)B}cm?4!D{TRB-1|LqfChW8e<~YGVOo*Q?BtIdfKTKl{dm@58 -zQAqx+kp3X^M+FG>C?VOzu*r*2{6zFP6v8L4kCezhqT&EVvL -zL~&Hdz34^ZGIy!qz~%d8K$7W_E5@aa2U*;NNhMPrZU<=BW#;8f&YM0dqGS-PCVy%U -ztaR<$YWAyjxxO{4#~YX;EbB`UIz>K?O}Ay0}1KdBm1!c=?ai96GMl6_@PCu)237UeN; -zd-XAKi6}I^_CjtgUNoLQC@*JZUYsJ#L;YMoThu6XkZtA^v&pt_mDyk`e&38M7i0vB -z#zD#ZQ*LeB|B{!Ldp@`J^#2mqk3@kXBns>!)S6-kpy=4>yG7qE`%-SL{J-R9Wq0M) -zp8Q|p`q{EanIE>jIK@1|mcPmzOE3?%9r)C2w;kPT9%wuNt$Bd$c9S_os11sJ;FQ@E -z`B?YXQS?%3d3^px_5(ZjRQJY2}4YWg+~tO{2}QnXHw@s;WY` -z7=A2t%;2XhCR|LF8Nu+`aV4fIo{JI3VVg;PP^PU4*%#HEaC)LM%VG8tqY*9%_(9xg -zsM%G84AFvCk?@bd?HLj^L{pLlZGJpaZO<0s|C9@P$wIO@r<<&XAPNq=1=EaS7y;8^ -z;_&8@c)=_P_7Ot-I3aloMMJX_DRKS|jnDZ=#FI5M@hNul3d0-MJh+K(+%t2&XhVnf9VQnMTDs -zzr1NgW>m~7rTNmN-jjxZprdFpbblQ>&1)XRaR+0WnAMAqJ3W?-r1x;k -zV~Nq@&5dmUQ+fs(7V`XNZlip?9eeSKaEWjk`E0nPYY5|iVjEQEtRT~Ae`uCI#zu~p -z(8KjcZoJ!dK&E41+y*-9)qmytbqq1~+7|+@hmxO;EfiR!U?4lXuo)LXD@bvw4{0$R -zfe*}#jyI!e7)#}hSJKh2Q$BiTWUvr5Oh~XeCNe{`^*Z?l?AyUG=wy-vQ7_i(q@}*F -zg>b-t80`X|hx7j`qsW*3f0%=n@xCwYuW)$Fcn?mLL7Et%x&!-`a9IUr45*2Z0rw_7 -z?Pyd3QZaa0y1k@>8;%1qmJT0Ew+FnEbQs%KzJ(H_OourX%(7LkWLr>KpkY}Ffir4a -zNhV=-HDR#uW8~>at{^Xdunk5Aek=XdQdScm)l47p^CqdhNgT?ML@A#vh{~Ir=)+U_ -zd@d_rdK2aI)hJNjNd?jmUrydKxTG>!xb`yjUr1^r= -zgV|Im>g)}tnSyXUF@0Y+i~_Ror@i59oS30ND^Qa8=2Mh{GxAMH(4{B6{2=C{75X3e -zn4r@`XA#mu`dERul!fx3PhM{Z585NY-n<0Uy-r&)7grVoYN6!-n=3VCd7F#K&hWaF -z!jfmiS0GA}U!h?dcmY>ZZ{uECrD!5NTQ-y)w)Wn(VMsmjK%8>`b#s5_$<$1X4+*>fhU7-_S0+7J^ -zm+t~u%NZL`Sr}(JxGk@yJP(Imu2cZYFCoR_B?RQKULg5fXB8k}){pD!M!pAdaqDk+ -z>+a;?HG{4{07z*OP1wZ*WN*GL()F_d@&f=VOA3&t=|h!snfZ)=0*h+>P{S=PR+Bvc -z+0q4?;0m3zy9prgZsM%dS4OyI7g{HgaITFrl)Qw&2y4bQoX+~&9sBc4Z3oNqY5@46 -z!jj3-eB4Uovv4(&8;Gl!xS6;^>N?B$O^+LdH-rMX2lv6|mAmea>+gBBb+>CiqSY(s -zjl!Kx{2!pmF8<*jdw)Cm3ui%EL%D$jRa@!7!$D0Nyi1Fre*>Ta5&i<C*~JCP`BW{iiVWM?+xown1tHUU7f#$_~1w?OzGp -z`tOM|yRhV4c_K}KNo)cjC#KCIx#4VuLHze^hWUZhhf(j7uEmoojAgdxbzr;r3`sS^ -z>PE3UhMSgI7K8u`Fvf@>)Afo;*MMI(qyvvT^Hd3lpzC<0an7LDKnkId%` -z-DruG>(g~E?@UtN7Lyxx3_i$P_qT|gXMu8JqNhAqPV`g-H_umER3q*`%bB*#aP$j> -zwc8|9F3skg+iWV=7w(@?G@~SyALS{J6i0cgB3UjB=q`Tm*|MYtwBLs5xRd5$xq8!q -zy4}FFsXR&7pttQ02`X<>bpz4|I8E2^JVzRdEQjK{Bc~&AsRje2b@Ve7sqs?P=)#>* -zC6DudDn5?pa~ygd3n_5mJMrIV%SM#Ug}Q(^oCajL5(P7t-$~VH(O+Lz&8T~=E&~*W -zizvZ`d?;^QE{KKHbHk3z-&;v%{rnThhLJJ6Cy!yvExv{|iGR{3&%tdV=oy@V-~Oq_r;f6}2Xwj67s7l&5L++3J -z<)FXQnLQFY{qFfF|9fCJmcqw?8kQIJVoTE!qtx2;Bq|k~Y3XWd5+i5=L@;Xw+#SWp -z@;32{vm96e5xuq;7eMe?4w~Ob1?y{CzK@C|2!Cn^QT?&0LN7KpS#ovaG`g$0hJWms -zXhmhRmEqyt0ymz;DX~_9z5N&zflng14XAf&@wS(N3KSQ$W9X*(`UgNKGuWkxnAy&^ -zxVZqB9eYDslz~Na{au+;g+?0gah^r@KH_n1N2@d{+C#Gq&s6q)fiFQk{~vx;HJ3xhSpl<9%tJxfs4vYAn(x)hOr`exwJmZb}d0 -zM~tAKNcyoi!{&~WH?zZUz=O1}dZb{A`in=j^sbj^$mbAz%K5+;@Xw6bL7E4#=W~rN -z1b*~}L4?CjicBns;P$P#NdHnhZeTv-5l4W@5v{aqjPT%;He7rV&62D@YcgmjKB*YA -z)2&NkH@@D -z^HVl?($Fzq96RieWoQ&XY?AV3a}SYJrK0FcWB$l<(eFxziG-999S0r7l0 -z1LC=M0zw(XKOk;)cs}N!V~C+hKdKn^Kv0}fI*Su3DlimTjEt2P~2ibaJq?N -zVBy6%k{jq625mfFn{YVgGgz#YKf8U1akOTL8BD^=c3#~C4lbiz##e(2;Zi!jGW^6I -zBfYq~u~&YCTbBH7)-y`k{2jb_a<3n!X!rSVd3Y5j?_F5<=Yc~&$EEFb2;tAeVpaGAVf}b0frXD2(xQdAg0MJB -zSRNt7%oh?Db1k1*|BQmIL&)G&l&~YhHvE*8CnUfHMJV`7tzCN|Gi7f1m#)1eBdyoV -zr@JQq;vqp@HSTBl)VHO7b?xm!JcWrV`n?x??o|C=vWQo$mv41pFG()Gy=$+`bh-4L -zf$QG}a}rQX(&MV6!{|?nC96tv91)}@ePS3cjw*f6Xb(*E87+x=A$%US?IB@(Jhg3@ -zF!v#0alEknH$u!bA#oNpYz{TdL=7_tJG2BXMnUePuA&Cvoh6@>Kj?Kr;dMu^Q$En^ -z6fb$5(p9g6Z(+*U(be^_BH-P6*}1+0ydnDGAbZ()6;6T;Jl -z7}iZgg!MdiQ-qKfEX?gAEH(+t{cSOyTCMjCb%OrxP>;}KsDC}cB8L5ncTnYF`pLe* -z=7;O?TBR~ZU}^3h&tLvO$Mbu@3OgR*leqAb?*wbOFtoogFI0FoSa_`uSNy&;K={DG -z?flk?rJX2w5Cd)Q-YdAJFby%z!IiS9*J#bTHn?OawG%nBUCtve49JoeXXL?$EN2xqW -zk0<n(~uf{S?eTBmgv@X00T3ENa6$gKeJJ=G0=%-m~r3eRF;VH@}78MvR -zGan-l{^v8K%RO+>GCQc?z!l53zU~9vmbrZ^_guEz>{Gj^%Tm$@h;>y69ViS67e@9M -zW`+vS1Pd?r;XZ5%4G`Wjz(!vvaLY0muQoyQqU%y_XG-#_1#Wt}~b&wlpa&$NNJY5L{4$2uf2iy6CTC|T5{hM@esw@l6#am~dI -ziqI;}y%Lm#2fhspezao_J^e776v3z*ok#}la2#Y~c>IhOaiJl3F7_CYpIsb&AjZnD -zhh%oL5AIZK58!h(|J!qf&(VsvcIOP0pzfk<66M$M36m_ol>ZK&)}f(n!-jB{y}Ugf -z0$O{(SUV6(qesTuvmbJs&&}20H9L%m3Krxsf3%LX&Os{+BtqfhjitIh5%Q#IIjBZa -zU`X3s-Pst*!A(Q%TAFB;*o4drBV;ul;;qsGp)sQJ-tyro6eP4tJWc7SME#(={5s?% -zI1D5>(C{s&6-Ti*Iuk~Rr0PJk2ARXiEp3w!7ThfVQd{=D{4H_$<3qM<|BqEziY=?h -zi|o};dc44Pe9_}~*1ykTqxN#v9wsZ-$2u;`n%UFJM_6ACwc=&?znE_IaDS7K|041h{z4%1#m#AHA2M=?C3P#kKqi+BmcK^ -zf@~^|OvzK9Ukt*z>XlBsf#>-AR^-t@*XKV)E)H~m-WI73^nCtf&AY&TgaWp`PMajn$QKd>wMUc>`Em -zrl&pDPjHU@+kQ&Y1$vLu0ohRiTyhlR_#66+)~GjyaQSdC9S4 -za;?~dLRKS0au_Dp3K4i1*OpW)o!6jCFkr(Li5X!VDjgy~`G~}}7anBYqGb>5fo*sV -zne&HXxaA5~TiYY@9~xoCw`*|utPBwT1kfxAeVr~$73UBZRQOhTC>}lASGAu}CcvCU -z)x>rC_U(u9ihUE;?Ty54pet8!s~ILBp&``SLWx>h)Gg!I)z|N@uUFTPTPJC#fgJ!N -z>GJwFax>_OHS7zSKhlW$nn_Cn^rA}Gut98{np@a4-oA5ya-e#kyuYZIDwlbca}^`C -z6C#uNgi)H!wzSNZ+iuKN%s;8j)ofIzWu{pczH{rb^~5hcM-3o&?7@PE*^LDcGl&IO -zPi=MkIiu1%QbU9^G-U5+Yn0g<)7LuNU|rnHx~d0l^C8wO1-EQp4!< -zqkWRP$Fkgrwwg@oeQ6cHwXS}W3V*_`zX$uJ$gvvuL7L`i<_{pn9(-A{jqtF -zN9XZ&ie+H+cI{~{6$3tTR5JSs`wnv`n(iHujBPi@F1FJe6=|IvZe1K=T}3s{hzhXo -zQm>3s)b7F{>#`q|Y59+C*i{O{P^LUOb;mJUe$pO9_YN}lPb602Kw>;t!>2D;!;`Gj -zdRt$Jv@Q>~M$bTx!k@8j8G{~$kB}fIECCPv;oyg3qI5{)%dS&X{ii|hGHX+ -z!(Idpp7-z_-wCq(I;NE%iFaR@-?ISjjhqWEHBM|K*A3I85ebGAI5{^-Quq^GwQrUwID|(;#yn)% -z?6!nKkKM=7-cY~@tqEMgg|*+(S^xy_uXP>${P5k)H~&E7+?!IYG_3O45{I24ruAdC -zCPnSHICJX6!4QOEl|X25^xz{FD_DcIs|eFxq5QJY73!duG2eTMjuQ*%I5D4&6LYNx -zW@E37wuYOm(S5ARQPycatuI7amxo%{bz|TCL`KG+egR}GZ%``iwZZ&i+Epbd_Y!!E -z!jNyJ4OhhmbGCC5QuxcaR5zt!jr=%CxCwaX$s6lsov=D}VFX5_*SSi}7K}51d_2L2 -zfyG5c96@`F!b45V^I27fgNe1I_6Rf#UFm2e0HkR`Ja)DNA!5xL>4s4w(=UpZ?8m85 -zdK_qK&J>5?Rfk;wY^0ULBd~EFJ7Wt6Id}nY{Z>wJMV@oW&LnhjJAHNg!?=wN0gExD -z!ha$H#5ej-+iXJv_u~04?r-qhk^}cD!F2sA+}3qLR+yidBoxZ&rO#sFUGl|n^zp4d -zB4f_IoaF3G#knquo7|wqX;LSofh`TL-)H9%wvYxIJt!cm8o|yRMOrPNcID?A&*jbL -z4xFN`Le^O(-eahce{Sz4Yt$3A7_vaEBpQ)U7>=w$X9E>yf{BN5AQVFUKnH~~{M?q$ -z1YB#gG!sEGwNu6qI#36Fdl6_sbSOk^zV=56@`F|w3&i0R{3s53Og_y|IW0sdxc7bd -zMfj!}1TrNPlwjEDByd##PAAIpZ0T74;36xzug;B#QWN~*Add3=xFK~~_cB<@;7*<+ -zO9wy=7Wq{Ds7+87yWh#*C1znP`uy-)=YMES(U0g&e2+%{?R)<8)(mL-8XcZnw5}v$ -ze6TV>-Qb$fMaFcJG$a4bIS~HhF*kyMDB_-smX)5+F8h0?4Pgey?pxc#qFuZ5_k{J4 -z`l$@x~S$ -z!JvKE0%AN!!%+0b-6*QXUmr-nUDipI+<+Flvt~nBABVGjdst7G)43nJj`a*9UhE(q1pf|pEHrE5gS=e=j8*CRD_?1$6$DPWaQQ2!*7`9WUh`fG -zG$|Py-_S|0a!GjgJ51Px4oGK+mpj&9mCg{P=KtIxSB7~8zwhG_8GW!7r=FV_iW{*3g&yd{ -z^Fez4W9cKroTELuQJ$;pcBt88lYUKhJ9L7iI>5LWa=z;l;CgJ|0M!-@WsIAQO}Giy -zfp^DKcP3mXQ3YY5WKa?9s8pqsb@DY2Y8Qwb0PO_z+wq0H4h4_Uu*gZ0crd|1p4}Q2 -zEaXeeg!c$bmuHOc{}$!i1NlFzyX0jOHF|y-pE( -zQar@C3YfHNcY$KbN%!%PoQi1oR0VhLg1bQFDp0zQ-xNOT!i|*AsHbfF=?flb4Kg7! -z(o~h6Dyq1vzW(5DezSqJb0m8Z!ez;Th>zxh5k23K-Nue|=2l6f -zfHYotxRD5(&gHcNke~??FX2kk1dZGTd!+w}`B?@x8Y|?lgQ_X_XZ#2SZBXesJceb* -z1PlLaOV>?btid^!DUIeoZF8vd-#23%4L%g_7 -zj_2&uT!hH+R)~vQx+j4lJ=7;MDT$o}m&HyLxN8HdUj)CQV@$E)*c_Xb%A6^$` -zoo$As5M-)UT;EplcjxjJ3nNeAxX3Wr?b)wqI23zV*xZV}{a78`Fnj9`tbGm*>VVt} -z%(JS*^bRhr74x~2j1lU@>5VYXb<0O%7GFP=H%Vw~L}~n;MxlwOl*XM~iE!>dg^Qhy -z!T_m32$%F1uAcd;fB8#G65+qSubZB*Ahg#H>!n51&L_z!V54iE!!o -z#51nlYZPD15_g@d3b` -z!_Jl%$s7&UFUpofWLW6LPA6pHR={U5e&f0Hj=(bU@0XCVoJy9;5C;0^9q2H|@Ae-Q -zr2wS!`NeTx$@u_&>;$$|zKI}S__2}Ag*^<(3O=>v0qPe;EDU}qbtk^*_JVHCE2^9; -zDqxk=k+-q2psPI|hyaivr)i#(F6;6*aAf-(O@T0gj*D#qM4@Gan{L1GsDKf20k9+d -z011Kl)ck-ZzNL{0>Uxj6&;rmQaOGz;LA_;RGk}o*OOW7rW)pX_F>;r0?!@1prxRdT -z`;qB%6V~5Jf`L;_;u9Tmzpe3+){I`(q$sP!XbtPj!qA{8c^#o1SnPw9vFAWpVNL3++guz}^0^`3pzA~MTm-L?l=EqPqeKnj5ek?digqFEJtvcBnXho72IuktHgb6dH*J}Opg`wee5O-*>x**U^jR5ysGz|EPGT6xhq`#k>1#-1WjM~9``iKX%s~`3Et9`;-H@}4%|I(zZ#GwnF^@HipgUx)=Ci*>6vz%a1LF -z#=-CRe=*k*;*j2Q101cwaU1X -znR=$W_tLKY=O2JT2VJ8#{8ZS?>GaWyjmFD~xF^yfe$X{K`2U3UH6BgI;Ecb9aog;- -zlSZULx -zszfq|fxb{1>Qr9m>)iVDG^Q-;b*|#yqEVIBvFQ)kb|8N>y$)cc7c%Q`b=PzF;6fxm -zR|k{TdLEzI@J*{@7sEHL&g9p;k;>2c1H->{hHw0KJRU@q%A|U@DL)O;0U7)2;clLn -zwmBMK{?a423j^P9RG#BFOSQ?bw^OEkDZ+R+E;l(RW!km~MNve5R#gertqX>sBewpm -zh2oZH3u`VU)X!sr6P(Wday3*B3@+}Q$A?^yLi~P^93T0!BaDHM)e%E= -z2r9V_Qm%Zw4jisU#V@PrD%1CHDsisjIGlMffp(RT+QOp*?Xs2^|f=`PHLrRI(5qUhdiWO0}nbYYJWSRBR&A$*ZK&0?@E -z+;Y*+gSI;$4LJ*G!`PkaBJ%s*;R*ZArrqX4rP7@i&$yn4 -zVJ~{eRWJb{4i51YXgyPvBc($O>aJ7|OdK_JV%DrzfXgnQFQ3}*N*$y}ixZ(|h~%NI -z1Vv%288iy&~K{U!rgqbaNHpKziPIUugB$!$nE;xC^xIu6nYVSwlvJ -zx(k%3{gb_6W%^2%KPUAO2D)(RRWo0F5M0J`-^%^_3jIrUN&Kn)?{`@N#~NEpheI_} -zVm=sZFHqX2YKtP`O)2|`a8RRa2p50u^j3!)oLZ8&d4I9KCb-%_O?7!VWx6-g(`A#peV(qF1qz{~bFAKn1J)wHFP^{VkH-Ty{`f@Zd*EH& -z`e3Akp#=~_4D6G}YtH7XmS(R^$1l?t8G7xzYQOnSsOg_o^%Y*282KbhTc$VE+6uq( -z)q{ORIarl91a%T9TDko7UVIdemH;wL|fm*}1c-Xy7vw5=VyS4$%)>8kl2o)@aKk -zZ9VX0-50S?ab1rsIqH)^pqu9%!$Q)Mt&IR6kUMw>H*F;p(rx{>)uODahSHL(Jv)gh -zinitbWLoZIRWs>9VwvLolb_-(;q}*)FRDg{FB}oJ@~MCUVI$&o{SR~jrNo;-8K;+l -zb!alpBW$2C<=i@bfQgHn$3`sGu$p<#t&2;zWtum{w7fble@MXQpnuR;*y3rv3W3JnmV{TzcAJ`Xrc51>bixN}56h;ka1}l;!-` -zlp4z!R--hzq;abG<8W($CWmXBdj2?tROt5BnRw=m3dQV8;{!_?9WiVG6Odra-yg6j -znjPITBh)FFX2kT08@JD}*sWIPEh*Eux2m|?S2jyxWLL>@W}QF(kDb#;s~;6G^icr= -zd<8Ih@4D5nWy(}U3%9R$)QY^9P@r1PxBCkGh6=3ojb8u@>!;DIdh|k~U5i4yA;tOe -zFlT$0nPI8Y|I>swvZMKwin^%&|-$~Z!NyY|4V;(>zGy1X`=LjSydK=MX$RS{n6vqcx&TYPuy%>wAxcU -z#k**QXVI&Uqo*8=o%V^dq;aLa59j)@<~6$P+w&I0&D#``^Ok9&B0Vs#m}0osnH!8`_8?0mJM=yHAEP2 -z>j&fMDxZ#-&)%s&9rnf*7rJ;b$GtSeyrW6!Ir_evRkh-MXuPLVVZv`KtMIUl?1r|^ -zUPJ5AeV%Trwr3bvSjx$!5uAkv(VJ0OwW7l@IRn>}Ico3dd -zdmYTaoUJ=_hKjV(SW8^4GdHk)r@cPczN>aq%03&nsty^L$o8S@AM69NM-}@H;b@drU8xZMf~yW-5_hu(yu2fx -z!Vj+laT8(O>96tf3Sjy4*Epko^)*&!pse4pn-TtSx=g_>Jk6~jlR5{!aiH~q_C;p7 -zKZsunlSW9vpB7(%EHI{myJL-7R;CdC_DwoiJ$ad+wlCqGc8NZ3&JQ5I;mO{j^6c -zTRko^Qao#g$&3uA(`HE`3igri%-O@%_(Z5pXQWt@X2(0%SS`<6!xlv1Y=@J@5gZZl -zu_*(w*=Si6{I7++sCZqyC}LDUfE?mxg)UNP?{tpUTHZ6QXCT6#KQI&qPkuV0U`B?VNs>FSR)+2fQi*Yv8Q~k^SvsYArw2yoO -zHC^znglRrYQ;y%y_OOP5R@c*-6ah2KR(m&g%ZRWbt3wMBfiCL`m=gdVs4hD;z?5uQ -ztkv?QHED=-*)Xdk91<^J2iX6^wWpp9GVPMov>eWDnhAeGB4fDr0283m{$d(MnKnu-o+m -z^Q{uZwq^zYhi_c(xAMlb6p&VpfZu;p99kH&tsj$Gs!G?*RT+jZgnR%^*IX4F4t0}F -zVqc}h8xX)p+IJpeV-3Q-Qg3Oleb%9~%lSS^T-!Vdj&TZJXKj^d)}h*Pu1{BW!iz+X -zd{R0>P@65@Ad6+NX^6&TR>KUOv`*P@ebVDLB#Oer*#OhJ{H4mIZA`3f)rt+bpjxG- -zX9t5m9f(j+*!)k2qeeXgc}rlHIl|M>#sQtbn272x`Vfq`2iFYS5--4giSTnLv_gDv -zAhfjFqIBF3qIgXNRM120fCus7H?>W|pT4Qx#2@w6js~3zBNNW{ep{8~4UF5k#hx-H -z>9cz7_bq#4*xrr>=0e<|K&yhjTVpGCMI^gxbaa5J4G2hq;tI3%;uqdxMcyXvB$d}& -z3-)yoT7mxYv2CEO4}$owG!8qHjy_N0uu~|!@9wwC)9)=uzw9bJ(X+F(!o6UXV?lP_ -z7H_|GBgU$Y%hyA~^qtmzZyMLl_>Cl0f}t0hM3-^`bSJ-jK;{l03m2o1hLXk-ut>Oi -zs(sNk`=aUgMKkP+X4)6cvM-u#Uo;0`DX#cNSV;tKjDc}9fAW-TDwC_oZgLif^uN?! -zdZhZMJ5(|(z0ac_q}#V&jngvrO<~|gBdlmMg7*|{iBO-cTQ``6wFcU^v+$;Os5LD5 -zEVx9ZEzwXd4*#Bbx#VfqUJJNeoqKYlf{Q|69ggE=s)lM|AUw#cz)ejE00FG73NEm-BJRf+(U0v`+0S69t4z4@=Aji(#lf#hn%EDX@Sp -zF#XND>?H9avP^Jig}buC9?=kZEzxnZdUy(AJX4>HH}{lb7gl*P!k7^nvn`@8N9=x~ -z`eEF@TPE+Trq)|{z3zc#&>=4JqQuPfpND~SMqZTq{)i?`SQ_#?()yw(d0nu2<79E= -zoumv6y-q$VNv_l4z&##w9q&+E92(r(ePsbO9(HCpG!BRQZQZ1j)bbFw1IifllUUFP -z+!<=1J67w;Q0FEUD>r32GpkaEL!`YPgb1(h?SuUxK|Uvvj?hYp6J -z-qN#Yo>o^~t-2iMuQylKf23lPWz$=1j}7sV&P^SuM9qUW8@q4{==jo}_Z`mcWs1pc -z_9c5(2)E!4t4t^D1zgg+nD(r&+wO_2=FsWCQO{K850Be>Am^t9GL$?S&tV$x -zdL}^NbIR?wduskhYxcca>cV6BMRCrdA=9HD7%XmcSXzqvxXGOrw*H4siy3I!YYxL@ -zGAp#{j5jM}Jp{cZ9LMUP&*SFY0lv@j047_cyi=oQ+8&QzgSByb~zX$c)tY -zhT&VR)R*E@LOFM2$R%gMovS<@sKB>|5|=BFNqqs`0!79@R@O*LJHYY?hRK^j37$&C -zII#f}lV)Wsn*VzI>&QuAYy`N#)@5p|T_Hcci?9sOeV8*)nTCNs0Tz*Dnw!#c8y|Mr -z#o}|}_SwY(2L-PnJg{4`D|H+GuI9b%J|W{ctV-k)-n|Dc)F!U)*CxAj&q$59{sDgq -zX{b!(ZRZEd=W24>aTlF1QV5lZ{fLi;Up12Z=||cm@eH-UuhN}$r(6cV2q+Xd6ZDsH -zwaX_T1;ILWJm#?X&QRmQoj5gPZ+remiJ6^){q1=ub+AjYwCCJlSFh168iI`nP6I+A -zh0RSPb8>T7R>&a75|$OGl4CWl=;SC575AY)ScjOC&him-0bNq37Q)G~x=zhWjyZKI -zv7a2J;w8swkPD+9&_xYR385$hLM)P_#ynw&kak0goWsEZDn7Q{NjK`~9=lqc2#d}7 -z%;~TGB#k9--6;hP0>*}){NPh4%3$zUJsA6#th1LrpNIB-KIZbT(?NX0Dc7@c?I{oLftE%sV97 -z)nixg#?XdPTFjZdX9Q1TRiJv#BjUzrhs8T12t0|(0F2(e4v^5`%h8bM0?~xa0w-C`#i~%v+&xQ*Ld_ -z?4KDW)%!A$$aKv_EmCH8h)bk$zBgDFI;1NYjTO#gIvDHBBo;XNNkSU~V-Q~Nph5ZZ -z*L89Hztz?2QC(0kqPm1vrJ62LO&~}4>KTnvpY_*+dYj-+_xsr^($r7xAd$5?6*sw~ -zPDK`Qgfyg}b_HPtxC4`nt3Z|;U9qn;*@-8JpJsQ}DJ>JmaJCM%6ufYyz<1Os*+b|N -z-@)#ZxZ|mElu8_rA}NX=PnAl2MNi)aP%iF(;{((xIVu2xi=^JurH<0YP(fl+2RT?4 -z4QUNbHm+f=M~}YV(wtmhJIym;tT%faF>lJ-XDnGnt*8Z<$8j`wcf!&Y~1%Q1_6Do6*$=X};2g-~F<#qVUM? -z4~MgA6bifnB%v1Fmq9B07|Vs)_>1^a6wm$R@p#{5 -zAALhB>=C?XF%uR362#$h3B$0!v+rVk?aG945N-a8Tl#1EK;OAPU~wSO5n5*CwGiBa(y%H;R+>6#{m5%QSz7Cpd~4k@VeIW>2)$#r -zZ=3~I2lW%Jd3e7Jv`RRYVjsC-SfYF&tr9qdeqCky+Q?s}HqK!?(3Vnu-I1g!M$+l4ahMJA}im8H#im+e)tmH>tOpT2KRHsFM+H) -zQp8Y-T}aFCq39THmlTeDkA1)okdyXD8ezu`F#x4-0wvS&TPe$0`uXz>gg*N<1W2|a -zD~)U33rIijsuo889XIc4;m6j9A=Z8qt;3$Mr9`12X+*cnr@ZT_GKk&6Exbi;$ZEhX -zz9}_=UyY)Lw<%y9rM8pn8LUiHnEJI`S7EYvuLHKMo^0p8>L?tIkv4Eo<7|VJgW4gm -z6%IF!74(t!9Zi{r)dTZpcSsJk_!dDt)E%2u*VJNPhFBufeKL_5tiGKbOv{r$adwT) -zFo}o_DQrOMo>9i}*aA&)w>E)f)qk4F7Ow|O4)Pen{EqQzy8q{Pz#c1t=oBFAgaze= -zj?;#fsoTG#X3$#wE7rqc^;pgAwC*Us=4|%Q>AG18!|e51#+KII9yFnLhpSQL(m6^L -zs5bllK^za+aopz}YjiW-PsbL#POcxzTDd)9ME_}#wE=~YVeAPuhEfPVTL=02GPN)Z -z>usTiuHcVXgv#?}gdn<`bafIa4F`D-27cR{2Xum!pT -z|3rJgz^8{tWR8{*37t7&Y(wcCXcebDTZ1c@vlbL{ajG;HDsd%^ -z1tO_dm>H+-{s_^X%H?{o?`g|`RdIiMp+jw>4OD~6uE5Vs}Yv@_q)sG2`< -za>m33cxdgtd{b$@t4TF~QhLTD9svjPO%?gOD(0sx%ShV<|KWU7ZT_y>`IA>M_9;!^4p_~AMw1%thtRp=;CI5F4G+TZHW~idL -ziL`2;!3w5w_|H3vCiUaTwX@-{2^+oTtb=R2OW!c=`rW)M{J67uyV&0O(Oa-&=`s}r -z*+8J=6B6Zh6Tr%L>>0m4K?1tx(Q*;d~w}0<#ygr{2$tL9c)<6rm7L;8mWa!Dc)aB{QL#R_B~sa!gn`~GdFC3paZwI$+#fz -zKx%lnu;6SZl#ID+Vk*5P3-;@Z!azCVTTMoT_dV0g$3i`4 -zQKzT&iTYYkRmC(M40jQ+25BDUoteV5Ul)G%<(28A;$^yc>4op*&zmwNv`Rkis`QMF -zPv`@+$jS2%Q$*MV}1(Y4ie2 -z^l#t8`x!SS7uRBzEM-g@YMhUhKOrQ70=1e)&g)|!r -zu>@N`*I0TtfQ2F3Pgn6tio!$X5yk3H4h7K&@(nlEE9e$z5YF82JuZ&wHR4y)3XM>I -zgO(6icVlykT8ek4`0M(gep$W88|WF-12Sf{RUd4@qikY> -zd{^Gh3|u?0VAgI)9wJUU#+C&@4rq|@q9iqOi6%*oBrG0?$k;lyX!AwFU_~V!D!p2~ -z7RX%zaVY+^ix0GycGna%D_j_MJ)=)BR7hH@ku!ty;&-sygtBM28@7=;`;=LMtK?E0zpB0@5l5ug>LqoDv!~|Lt~-#O -z;e34yE-pFsC7G2FE9U=Hk2fH2^|&I%_hi%w^QcdQ5cTob;5&=~pj5&$=YAQa3DC=* -zWk)vGDLYtk%YC^BPni~I_PB4q^H#^TR?ou|nn~4h((XEM-~NClMw+o2wU~Joj@X(A -zKJ#jMKT^%vvzu?ba$J&l@TLP$8D+ve8oxeB2b3>1(#ccLRpA56Xdi(yTK*irtX`Ip -zI9Z&?Ev`?ho+sxcMYsr;b9tq*M%|ziKLKiG^)+hl;eC4Zf|djb7cwk@V&wicy*F+` -zqA@VTtIl|5SWaeJPG+Q>8ZICZ94b6}LN)>X@X{Uv{P2fy6!2UTMU9c=jf58zx) -zPN5y=WI0s4M@}h}gENMmBPF;?PAQvs=K-!bQYa@m<-%_JWGAv1Q%#~Jk?|xds%jEF -ziAw2L1-6B0@34LQKIZ=Ou0#{g0)55QKq|XAJBVcDa*9fPM$aMkrL5rN-%YSvmFN!GlMGDU2 -zihb!&XLfFCE?y58Ixp4fzK_z#1v?=OfwF}!ec379l|63_-?Kh`#`^#G^ers!hnLcI -zMN!FA6W`uRVno}PFtvs=qZO=i@26~Vomx2Vt3@so3Ys`QMA~3NKx*}5;pi{orLh*; -zF*?#%WN-dteOPwQh7G~RYxisp?729YZVV84lzQIhW=~stSpc~A|K4iuv#w54yHSpj -z%IQhvgas_-{(C*yLRZGNY@9&k_7LCx-|A5C-~Ly1@KDM5McP8v6Q=NnDO6ZN$alP}1FdU-B}s_%_r6YdmH(5!{Gb1a<^QVT{_=?~lY;hN>rBUBqy0A? -zmHz@$8-8&ZwSV=m+P@m0qHYb-w1uhov%eUSN97VU;pu33v;d=$|EFL1?>_RQ?xi@n -zh`;yA!@&9H%aN@?*09l53x^AM_}{fPaqKq12crV`5Vg$V;63u8BUot>a)Y>vxWLGq#f -z;25@KB&l4hi{zw? -zs>w=8ZQkO{k@Eri9s;_gzJmC)>q;qdVLbtvEBJydWs2j~tUed&Xd`+B8c2|f%!Ixh -zE2ZbwU42`Zea}>C%6G)&=Z3@;uM={v%H`g9K@tlTfKW7G>%0&*BO|O#BZgjuXkpwT -zQ<)~Ad>!BYDp8p(P!pvwfhCm?vZT_7EUB2S2VnYzs8)RS6>1@xdfev_4*MNPB!`@z -zHX!lSE7Ip*^2b9Gu}bmtEB-oeCv~G~$ElHxf(0-k)z@AhtUOCzYRicC)q_>*8OGEv -zS!QBN@MK)fZpL_lAOltj@IkJWoYL2+DUQb`L~*gGbA4A_Go)~^&l$u@1R<@@A1i+9 -z^Ftnxn#mvfmH(jZmveKi@JVfx{IT43U4HQM@P;tgLIF}+%9Kv{&-wgC;r&g?PW@2R -zP$yJrZd_xHBfwgR5j57|b=vt$*SPPm)s{j)-f>P4uhCG;U_)hb53l7N*X$V)yv(ZdciZ1>jlEQ -za7{u?m!QW+Hvx96&ov`M&kE$&B4mwps_sTcpHUah*mN -zqB*Qm+b-Q}p`8WYJK^$m!VSKC*gppTuh8d)@UX@5QI6gDM$V?>9LqF$o(tczhO4q$ -z``p}iiuwLI9Aa^mUw(sX$aeiAa79<3yv}slf&By3*-g+z`83{mr#3$O_IL#|uPtHT -z!YWo1H{Llx4YHxevy~JGVBaFnvO^<&-m7uJ(%%oSBDvg1xa%H*H -zlI);`L8@#o_Lv@>k>7*J%BGDn3x7QV>7F6_J=ix2j~q-ad1cYm>ECO!Zyhm`oZdvd -z0=DJSkO_(G@Vi;Luf%m$#pPDzJhzTj-+XpmEv%hs#Br1)b$FEF)tRAFrsqR^8(LS1 -z6lk!;;tHC&{i={e-rQG!y30#;Of3fmmoX&Jo9H{^c212PcsqBh@m9{=gq)Nr9C4Fx -zOjIEG$gBo4tF1rfYzS@)9e-9j;}LEqnz7h?@nCp~3VIPkZ65<@WE>;@@dj|m>P;Yp -ztusS-Gsog(2jp1t<#-Z}tyJK2!|F1s-!O0)pPNg8wN{xSj)Ge -zHA=(vn`Qgbnr}Am+YyzkJ9Kb!>9Lxl$_<}F3ix2ynO&DRTWrsM^>m7qrA`fj8YP!= -zq(rsZk~VaPqpLljJ3?|Dx-imw01uI02YX+VKMOQv!*UXid|k230}>FpH+sJQqG71P -z_Q~*|8Sg6h48tB&seufn-Zs9aMj^g_1ahkyT8}HG$Ca>Fu(q2ny7zRBZt@hv1O97O -zMKX6ry8020K>12m)iN&B^uCE}xHwu;Ab|mh*r9RpW@yS(st^ISTDm+Vm{k$h_w9rH -z3peQcB^>SG>sv&@x1MkZExyhjZy5}YdpZ6~xczegyG-cvKkJ2m)ek0nU6u9PVCF1_ -zfjd~wACkg^@_KC`W3w38D6yg*bl6Yog(^AqKkBj19jYg}K;_F8Syztah)pETY9Q6* -zq(0VVeXaH3)3Kx -zkSWo$2_FjJuBkwA!TC^}c|T#3(=aqQ5Jng+w&W``Hk3yEgPewrlGFc-G&J^sH1=J3 -zR|Hc&nb~-a$?`~Mwg*g!P)eOO$uBFU8H|e{(u-f-6=r@#brQU?dx2L~qT=WHB&|!S -zu`7cd&cEP)p?2f30octyqG%*n}F0Eay(NE!>ku__5^oj>KIgd`IM1REZzWf1d+t+>|LHV86Tna7QyXTdFfGg?|S0LUBgF!)(VHs&hQOfOst#(tQ -zSu)q2VDTC(R4kRt85=9-c%+dWp8Y0FJ-F^iTU&hpS!wLCe79U5i&ZRcz0MwoWgLFIyjsymddn4EEKwTp!l5nsk$emIDsfdQ!hakY -ztoM%x+e^QN>74-EJdoirCG1Z~P;W|^WPyo$dPi9vGteOeKxdR_oc-&7rrFp*!E867 -zJB)pRF+E85s9kc55zEPOTwHVAAdSdT_Lzvb$FRmG+vY8{Q~$XY()RvynN}Fx$g2It -zLa2g!_4@4D5@i!O%k72u*_(+k=X!E1%l++ef#0BWd_O6Wjg=&vjuWT)scMDGrH*_L -zrVbX`or^QPFh`n`V$Tb=g&Wx*SM?ii?hQlfq>4jFJ%#4jo%3o#$`Il -zH?~gE%Dpitc+&OOzcHM$iSl+w_|r!8(u3`zR%Dt8kwUj!e32NuL8TcU=i;gqI|?sT -zpzucTUlp-Cr!b&Cx8OxluArt>Vr%R;ue1YjkNRu@l3@ -z2-t9&tB;@LFEv);0Pm^Bd%a>EReL$Av)QIo4UxUoC@1Q;mBJ1ws -zHw-T!174WJdfJV! -zS8lF7Q+N|nYzi*Ze`j2y?EfBkE_%z$ub5Xx-;s8*_t}3RgM<3Tv6llP*zO(^W#+eZ$Z; -zuiAMjG0oEU<-XS6DqO#1S}rBF{#Ggds%*)R_}Br)+n_ef%qk(MsVFMO`Jwsw57